From 8c68e819b822eaf8f673ea1e605eae6e99e4bcf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 22:48:55 +0200 Subject: [PATCH 001/155] build: pin @convex-dev/migrations 0.3.6 The latest release, 0.3.6, has the peer range convex ^1.42.0 and convex-helpers ^0.1.115, which admit the pinned convex 1.46.0 and convex-helpers 0.1.124. Co-Authored-By: Claude Opus 5.5 --- package-lock.json | 50 +++++++++++++++-------------------------------- package.json | 3 +++ 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index 20146a6..f4899fd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,9 @@ "": { "name": "libar-platform", "version": "0.0.0", + "dependencies": { + "@convex-dev/migrations": "0.3.6" + }, "devDependencies": { "@convex-dev/eslint-plugin": "5.0.0", "@edge-runtime/vm": "5.0.0", @@ -71,6 +74,16 @@ "convex": "^1.43.0" } }, + "node_modules/@convex-dev/migrations": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/@convex-dev/migrations/-/migrations-0.3.6.tgz", + "integrity": "sha512-cURYG4uy323MDi55QWNF5HAGemS4QspSgJ4uP+gGPAJdBLBbk2KFO7toFlKhacc1NinJQiG6+D/YEFyFyBS1/A==", + "license": "Apache-2.0", + "peerDependencies": { + "convex": "^1.42.0", + "convex-helpers": "^0.1.115" + } + }, "node_modules/@cucumber/gherkin": { "version": "42.0.1", "resolved": "https://registry.npmjs.org/@cucumber/gherkin/-/gherkin-42.0.1.tgz", @@ -118,7 +131,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -135,7 +147,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -152,7 +163,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -169,7 +179,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -186,7 +195,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -203,7 +211,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -220,7 +227,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -237,7 +243,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -254,7 +259,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -271,7 +275,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -288,7 +291,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -305,7 +307,6 @@ "cpu": [ "loong64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -322,7 +323,6 @@ "cpu": [ "mips64el" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -339,7 +339,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -356,7 +355,6 @@ "cpu": [ "riscv64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -373,7 +371,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -390,7 +387,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -407,7 +403,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -424,7 +419,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -441,7 +435,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -458,7 +451,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -475,7 +467,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -492,7 +483,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -509,7 +499,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -526,7 +515,6 @@ "cpu": [ "ia32" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -543,7 +531,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1940,7 +1927,7 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -2449,7 +2436,6 @@ "version": "1.46.0", "resolved": "https://registry.npmjs.org/convex/-/convex-1.46.0.tgz", "integrity": "sha512-dhBwDQTZDOhi5SUGAX17EWNehCdNq6/UFrSL0eLZuGfMkdsQbCVWVRZCzbZ5CF+/pgvoGxtApbEoTSipqL4JTw==", - "dev": true, "license": "Apache-2.0", "dependencies": { "esbuild": "0.27.0", @@ -2488,7 +2474,6 @@ "version": "0.1.124", "resolved": "https://registry.npmjs.org/convex-helpers/-/convex-helpers-0.1.124.tgz", "integrity": "sha512-Gbu/jq6I3x5Y51cXP5c7+CI3t15vX5Xi2G0tOym3rxpBtYqrI+qWwxkqZgJYrHHccY42wZudB8h6kS3fak8Aeg==", - "dev": true, "license": "Apache-2.0", "bin": { "convex-helpers": "bin.cjs" @@ -2611,7 +2596,6 @@ "version": "0.27.0", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.0.tgz", "integrity": "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==", - "dev": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -3631,7 +3615,6 @@ "version": "3.9.9", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.9.tgz", "integrity": "sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg==", - "dev": true, "license": "MIT", "bin": { "prettier": "bin/prettier.cjs" @@ -3677,7 +3660,7 @@ "version": "19.3.0", "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", - "dev": true, + "devOptional": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -3871,7 +3854,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/@typescript/typescript6/-/typescript6-6.0.2.tgz", "integrity": "sha512-mbCddXd+jm7hfx7w2YU64/Av4/NqqeG3GoRZgxPcgoTxYjhrcfJRw9ULch71SS4G+Q3bOXFhRvPqjguN0Hyp5w==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "dependencies": { "@typescript/old": "npm:typescript@^6" @@ -4290,7 +4273,6 @@ "version": "8.21.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", - "dev": true, "license": "MIT", "engines": { "node": ">=10.0.0" diff --git a/package.json b/package.json index a0e7ea9..7219ea7 100644 --- a/package.json +++ b/package.json @@ -48,5 +48,8 @@ "typescript": "npm:@typescript/typescript6@6.0.2", "typescript-eslint": "8.71.0", "vitest": "5.0.3" + }, + "dependencies": { + "@convex-dev/migrations": "0.3.6" } } From 6ca24810aa4ceaeeea873f9d856ae886faba5530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 22:54:50 +0200 Subject: [PATCH 002/155] design: the answers in the decision forks; the largest order, the speed targets and the local reading of Sc L2-9 at their Specs Each ruled fork carries its answer and the owner's words it rests on; the licence fork says it is not ruled. The Specs stop calling 100 lines provisional: it is the bound PlaceOrder's declaration carries, the example's promise and no platform limit, and the open questions on OQ3 close. The first experiment states its latency, throughput and contention targets for the pinned local backend. Sc L2-9 reads as the local observation on the production composition, its concurrency half is the contention run, and no Spec claims hosted parity. Co-Authored-By: Claude Opus 5.5 --- design/decisions/forks/OD-025.md | 4 +++ design/decisions/forks/OD-046.md | 11 +++++++ design/decisions/forks/OD-047.md | 4 +++ design/decisions/forks/OD-048.md | 2 ++ design/decisions/forks/OD-064.md | 4 +++ design/decisions/forks/OD-068.md | 4 +++ design/decisions/forks/OD-069.md | 4 +++ design/decisions/forks/OD-123.md | 4 +++ design/decisions/forks/OD-148.md | 4 +++ design/decisions/forks/OD-150.md | 4 +++ design/decisions/forks/OD-151.md | 4 +++ design/decisions/forks/OD-152.md | 4 +++ design/decisions/forks/OD-153.md | 4 +++ ...acceptance-production-configuration.sdp.md | 5 +-- ...first-experiment.order-of-max-lines.sdp.md | 6 ++-- .../specs/application/first-experiment.sdp.md | 31 +++++++++++-------- .../specs/application/parent-use-cases.sdp.md | 5 ++- design/specs/command/command-pipeline.sdp.md | 2 +- design/specs/context/batch-shaped-api.sdp.md | 6 +--- ...03-events-only-source-of-next-state.sdp.md | 2 +- ...0-contexts-meet-in-parent-use-cases.sdp.md | 8 ++--- .../kernel/state-document-mapping.sdp.md | 8 ++--- design/specs/platform/native-harness.sdp.md | 4 +-- 23 files changed, 92 insertions(+), 42 deletions(-) diff --git a/design/decisions/forks/OD-025.md b/design/decisions/forks/OD-025.md index f0c047e..3cb4017 100644 --- a/design/decisions/forks/OD-025.md +++ b/design/decisions/forks/OD-025.md @@ -50,6 +50,10 @@ The provisional reading is the composition's internal mutation run with admin ac ## The owner's answer +The first grant of a tenant is created with admin access through the composition's own internal grant mutation, and later grants the same way or by whatever command the composition registers. The library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation: those are an adopter's to build on the helpers. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The two compositions register grant mutations over insertGrant. [example/convex/grants.ts:19] [fixture/convex/grants.ts:18] diff --git a/design/decisions/forks/OD-046.md b/design/decisions/forks/OD-046.md index 97be31a..cc582e5 100644 --- a/design/decisions/forks/OD-046.md +++ b/design/decisions/forks/OD-046.md @@ -40,6 +40,17 @@ The row records no provisional reading; the Spec measures at 100 and the declara ## The owner's answer +One answer for the largest order: 100 lines is the largest order `PlaceOrder` of the example supports, and the bound its declaration carries. It is the example's promise and no platform limit: a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings. One document per stream stays the default mapping, with the derived mapping for state that outgrows its budget. The Specs stop calling 100 provisional. + +The speed targets are set here, before the benchmark, as the doc requires. They are targets for the pinned local backend on the machine of the run, and a hosted deployment is another question. The main thread had seen three single local runs (51.6, 39.1 and 184.4 ms at 1, 10 and 100 lines) when it set them, and the record says so. + +- Latency without contention, the execution time of the top-level mutation as the backend's log reports it, over at least 30 accepted commands per size after 5 discarded: median at most 100 ms at 1 line, 100 ms at 10 lines and 400 ms at 100 lines; 95th percentile at most 200, 200 and 800 ms. +- Throughput without contention: 8 callers at once, each placing ten-line orders on stock no other caller touches, complete at least 20 orders a second over at least 10 seconds, with the local backend's write cap of 4 MiB a second stated beside the number. +- Contention, N commands at once on one stock item with stock for one: at N of 2 and 8 every command reaches a business outcome, one applied and the rest rejected for short stock, and the engine fails none after its bounded retries. At N of 32 the same count is reported, and a command the engine failed is a recorded cost and no failure of the experiment, because the caller repeats it under the same request key. +- A target that is missed is reported as not held with its number. No target is changed after the run. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The declaration uses the provisional 100-line bound. [example/convex/ordering.ts:17] [example/convex/ordering.ts:42] diff --git a/design/decisions/forks/OD-047.md b/design/decisions/forks/OD-047.md index 6acb8bf..5c03e72 100644 --- a/design/decisions/forks/OD-047.md +++ b/design/decisions/forks/OD-047.md @@ -41,6 +41,10 @@ The row has no provisional reading; the two references remain in `AGENTS.md`. [A ## The owner's answer +The two references in `AGENTS.md` under "On the owner's machine" stay. They are paths and hold no content of the files. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The references are present in the opened AGENTS.md section. [AGENTS.md:44] [AGENTS.md:45] diff --git a/design/decisions/forks/OD-048.md b/design/decisions/forks/OD-048.md index 9c7ae2f..b98ce1f 100644 --- a/design/decisions/forks/OD-048.md +++ b/design/decisions/forks/OD-048.md @@ -41,6 +41,8 @@ The row has no provisional reading; the opened root package stays private and co ## The owner's answer +Not ruled. The licence is the owner's. + ## Notes for the main thread - Check: The package is private; the LICENSE list is a research example. [package.json:4] [docs/modern-ts.md:215] diff --git a/design/decisions/forks/OD-064.md b/design/decisions/forks/OD-064.md index ce26ca1..2b0b8b9 100644 --- a/design/decisions/forks/OD-064.md +++ b/design/decisions/forks/OD-064.md @@ -40,6 +40,10 @@ The provisional reading measures at 100. [design/specs/application/first-experim ## The owner's answer +One answer for the largest order: 100 lines is the largest order `PlaceOrder` of the example supports, and the bound its declaration carries. It is the example's promise and no platform limit: a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings. One document per stream stays the default mapping, with the derived mapping for state that outgrows its budget. The Specs stop calling 100 provisional. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The adapter separately checks the planned stream count and budget sum. [src/context/adapter.ts:270] [src/context/adapter.ts:277] diff --git a/design/decisions/forks/OD-068.md b/design/decisions/forks/OD-068.md index d64d69a..34ef25b 100644 --- a/design/decisions/forks/OD-068.md +++ b/design/decisions/forks/OD-068.md @@ -40,6 +40,10 @@ The provisional reading keeps retired generations unwritten and reverifies rollb ## The owner's answer +A retired generation is not written. Rollback reopens it as `verifying`, runs one verify pass that repairs what the intervening writes changed, and then switches. The switch is one write in both directions; the Specs stop promising a rollback of one write, and say rollback is a verify pass and then a switch of one write. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The writable-generation reader excludes the retired state. [src/read-model/generations.ts:60] [src/read-model/generations.ts:91] diff --git a/design/decisions/forks/OD-069.md b/design/decisions/forks/OD-069.md index 06dc0be..479cd04 100644 --- a/design/decisions/forks/OD-069.md +++ b/design/decisions/forks/OD-069.md @@ -40,6 +40,10 @@ The provisional reading is deployment-wide generations with tenant-leading marke ## The owner's answer +One generation per read model spans every tenant. The generation registry is operational data of the deployment and its indexes lead with the read model. Every table that holds tenant data, the read-model rows and the markers among them, still leads with the tenant. The deviation from Law 11's index rule is recorded where it is today, as confirmed. A rebuild for one tenant is not offered. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The registry indexes omit tenant, while read-model rows carry it. [src/read-model/tables.ts:54] [src/read-model/tables.ts:60] diff --git a/design/decisions/forks/OD-123.md b/design/decisions/forks/OD-123.md index 7b3a7af..e454d60 100644 --- a/design/decisions/forks/OD-123.md +++ b/design/decisions/forks/OD-123.md @@ -50,6 +50,10 @@ The provisional reading is one document with the maximum as a placeholder. [desi ## The owner's answer +One answer for the largest order: 100 lines is the largest order `PlaceOrder` of the example supports, and the bound its declaration carries. It is the example's promise and no platform limit: a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings. One document per stream stays the default mapping, with the derived mapping for state that outgrows its budget. The Specs stop calling 100 provisional. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The example uses single mappings with the two stated budgets. [example/convex/orders/streams.ts:42] [example/convex/inventory/streams.ts:35] diff --git a/design/decisions/forks/OD-148.md b/design/decisions/forks/OD-148.md index 72228d3..e21d692 100644 --- a/design/decisions/forks/OD-148.md +++ b/design/decisions/forks/OD-148.md @@ -50,6 +50,10 @@ No provisional reading is recorded, and no rebuild/restore operator entry was fo ## The owner's answer +Admin access is the authority of an operation over the whole deployment. Every operator entry is an internal function, so only admin access reaches it. It takes the operator as a required argument, a string bounded as an actor's ID is, refuses an empty one, reads no grant, and writes that operator on what it changes: the generation row, the gate's audit record, the restore's run record. The Specs say "records the stated operator" and stop saying "authorizes an operator actor". No scope beyond the tenant is added. D11's one trusted deployment is the boundary, and the Spec says a stated operator is what admin access chose to state. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: First activation stores the given startedBy without a grant read in that helper. [src/read-model/generations.ts:16] [src/read-model/generations.ts:46] diff --git a/design/decisions/forks/OD-150.md b/design/decisions/forks/OD-150.md index ded96cb..0bd371e 100644 --- a/design/decisions/forks/OD-150.md +++ b/design/decisions/forks/OD-150.md @@ -40,6 +40,10 @@ The row has no provisional reading; the lean keeps the Spec silent on a duration ## The owner's answer +The write-pause Spec states no acceptable length. Before a paused rebuild starts, its length is estimated as the sum the lean names, backfill batches times batch time plus verify batches times batch time, and the number comes from the first paused rebuild that is measured on the native backend. Whether that length fires the doc's trigger for an online history rebuild is asked of the owner then, with the number. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The intended gate never reopens itself and resume needs switch or abort. [design/specs/application/write-pause.sdp.md:23] [design/specs/application/write-pause.sdp.md:46] diff --git a/design/decisions/forks/OD-151.md b/design/decisions/forks/OD-151.md index 08b3ff8..cb1e28c 100644 --- a/design/decisions/forks/OD-151.md +++ b/design/decisions/forks/OD-151.md @@ -41,6 +41,10 @@ The provisional reading keeps `allocated` as one total. [design/decisions/regist ## The owner's answer +Stock state keeps totals only. `release` takes the quantities from the cancelled order's own lines, in the same mutation as the order's `cancel`, with Orders called first (OD-074). The stock decider rejects a release larger than the allocated total. A second cancel of one order is rejected by the order's own state, so no stock item needs to remember an order. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The stock fold retains quantity totals, not order IDs. [example/domain/stockItem.ts:10] [example/domain/stockItem.ts:95] diff --git a/design/decisions/forks/OD-152.md b/design/decisions/forks/OD-152.md index 7cbd86f..e5c833b 100644 --- a/design/decisions/forks/OD-152.md +++ b/design/decisions/forks/OD-152.md @@ -41,6 +41,10 @@ The provisional reading includes the history view under a write pause. [design/s ## The owner's answer +The order allocation history view stays in the first experiment, because Sc L2-6 is a row of the doc and the experiment passes on every Layer 0 to 2 row. Its rules are written first (roadmap 4c) and its build is a unit of round 5, after the online rebuild. The roadmap's deferral "at a view that needs them" now covers the aggregate form only. + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The declaration binds only the order summary. [example/convex/ordering.ts:40] diff --git a/design/decisions/forks/OD-153.md b/design/decisions/forks/OD-153.md index ce5ebe9..db19c7a 100644 --- a/design/decisions/forks/OD-153.md +++ b/design/decisions/forks/OD-153.md @@ -41,6 +41,10 @@ The provisional reading is local native runs with production configuration. [des ## The owner's answer +Sc L2-9 reads as the local observation: the production composition on the pinned local backend, with the same authority, schemas, concurrency and code path a release has. No Spec claims parity with a hosted deployment. The concurrency half is the contention run on the production composition, as OD-007's checked lean says. A hosted run stays deferred with Probe 3's quota half (OD-036). + +Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. + ## Notes for the main thread - Check: The native example invokes authority, schema and code-path assertions. [tests/native/first-experiment.native-acceptance-production-configuration.test.ts:283] diff --git a/design/specs/application/first-experiment.native-acceptance-production-configuration.sdp.md b/design/specs/application/first-experiment.native-acceptance-production-configuration.sdp.md index 9c44ef0..b5bd699 100644 --- a/design/specs/application/first-experiment.native-acceptance-production-configuration.sdp.md +++ b/design/specs/application/first-experiment.native-acceptance-production-configuration.sdp.md @@ -16,10 +16,6 @@ Sc L2-9 · end to end tier · production composition. - outcome: Same authority, schemas, concurrency and code path as a release. (Sc L2-9) -### Open questions - -- [non-blocking] The row's pass condition names concurrency and this example binds no concurrency claim: no native observation shows that the disposable backend's concurrency matches a release's, and the test shows only that the deployment's environment holds the three variables its `auth.config.ts` reads and no other; the owner rules whether that reading is enough for the row or the row needs an example that runs commands at once on the production composition (Sc L2-9, E-15) - ```gwt Given the production composition on a native backend And the backend runs with {configuration: "production configuration"} @@ -30,6 +26,7 @@ Then authority, schemas and code path {parity: "match a release"} ## Verification — executable - Runs in the end-to-end tier on the production composition; every test owns its disposable backend. +- The row is read as the local observation, the production composition on the pinned local backend with the same authority, schemas, concurrency and code path a release has, and this example claims no parity with a hosted deployment; it binds no concurrency claim, because the row's concurrency half is the contention run of `spec:application.first-experiment` on the production composition. - This example is the end-to-end path alone, and its test runs no other scenario: the harness deploys `example/` once, with its own `auth.config.ts`, grants, namespaces, entry points and schema, and the test sends, through ordinary clients and one call after another, `ReceiveStock` and `PlaceOrder` applied, a `PlaceOrder` rejected `insufficientStock`, a `PlaceOrder` and a `getOrder` refused `forbidden` to a caller with no grant, a `getOrder` refused `unauthenticated` to a caller with no identity, `getOrder`, `listOrders` and `listOrderSummaries` answered to a granted caller, and each internal function of the composition, named directly, refused as not public. No two calls of the path run at once. Whether every scenario routed to the production composition passed there is the acceptance check's to answer, under `spec:platform.acceptance-contract`. - The test asserts that the function log, from a mark before the path to its last call, holds one top-level query or mutation record for each call of the path and no record of any other function, that each refused internal function read no document, and that the applied order's receipt, its order and stock item stream rows and its order summary row are stored and the refused calls stored nothing. - The test obtains each identity from a token the harness signs for the fixture issuer that the deployment's environment variables name, carried by an ordinary client, because a production issuer cannot mint tokens for a disposable backend; the evidence record names that issuer as the identity source, the one named difference from production, under E-13. diff --git a/design/specs/application/first-experiment.order-of-max-lines.sdp.md b/design/specs/application/first-experiment.order-of-max-lines.sdp.md index cc81287..9e33910 100644 --- a/design/specs/application/first-experiment.order-of-max-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-max-lines.sdp.md @@ -9,7 +9,7 @@ relations: --- # An order of the maximum number of lines -Sc L2-3 · native tier · production composition · the third of the three enumerated sizes; the maximum is the provisional 100 until OQ3 is decided. +Sc L2-3 · native tier · production composition · the third of the three enumerated sizes; the maximum is 100 lines, the bound `PlaceOrder`'s declaration carries. ## Intent @@ -29,8 +29,8 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the provisional maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, and a job scheduled with a delay past the window is not seen, and that the list the read returned already holds the order's summary row; and it asserts that the completion record's execution time is under the 1 second mutation timeout. +- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, and a job scheduled with a delay past the window is not seen, and that the list the read returned already holds the order's summary row; and it asserts that the completion record's execution time is under the 1 second mutation timeout. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation carrying all 100 lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, and asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. - The test then receives stock for 101 more stock items through `ReceiveStock`, sends `PlaceOrder` with one line on each, one line more than the maximum, and asserts the `operationTooLarge` rejection whose details carry 101 items against a maximum of 100, with the receipts, both contexts' stream rows and events and the order summaries unchanged. -- The test records the cost per added line, the difference from the one-line reference divided by the 99 added lines, so the owner can decide OQ3 from numbers. +- The test records the cost per added line, the difference from the one-line reference divided by the 99 added lines. diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index ee725a0..23a6f07 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -39,7 +39,7 @@ relations: Layer 2 · Detail: full · Traces: First experiment, Acceptance scenarios, Law 1, D3, D8, D9, D10, D11, F4, F13, Probe 3, OQ1, OQ3, OQ4, Sc L2-3, Sc L2-9, E-8, E-13, E-15, E-17, E-37, E-46, E-47. -The first experiment builds Layers 0 to 2 in a small, clean application, Orders and Inventory, and measures what the design costs. It passes when every Layer 0, 1 and 2 scenario passes on a native backend. It measures orders of 1 line, 10 lines and the chosen maximum, without and with stock contention, and counts top-level commits, function and component calls, documents read and written, and rows left behind, with healthy-path and retry costs reported separately. Semantics come first and speed second; latency and throughput targets are product decisions made before the benchmark. The results feed the open question on component cost and the Layer 3 decisions. +The first experiment builds Layers 0 to 2 in a small, clean application, Orders and Inventory, and measures what the design costs. It passes when every Layer 0, 1 and 2 scenario passes on a native backend. It measures orders of 1 line, 10 lines and the maximum of 100 lines, without and with stock contention, and counts top-level commits, function and component calls, documents read and written, and rows left behind, with healthy-path and retry costs reported separately. Semantics come first and speed second; the latency and throughput targets are set before the benchmark, for the pinned local backend, and no target changes after the run. The results feed the open question on component cost and the Layer 3 decisions. The six cost targets are the constraint Specs this workflow is constrained by. The example domain's terms are `spec:application.orders-inventory-example`. The experiment lives in this repository, beside the design, which settles OQ4. @@ -49,7 +49,7 @@ The six cost targets are the constraint Specs this workflow is constrained by. T - problem: Orders of 1 line, 10 lines and the maximum must each be one top-level commit with zero projection jobs, one call per context and budgets that hold; without a built application and a measured run, the cost of a component call and the size of one use case stay assumptions and Layer 3 would be written from guesses (Sc L2-3, Sc L2-9, F4) - outcome: Layers 0 to 2 exist as a working Orders and Inventory application, every Layer 0, 1 and 2 scenario passes on a native backend with production configuration, and the six cost targets are measured for 1, 10 and the maximum lines with and without contention (First experiment) - value: The measurements settle OQ1 and Probe 3 with numbers, prove the transactional profile before the durable profile is designed, and give the owner the cost line every mechanism must be compared against (First experiment, OQ1, Decision method rule 2) -- risk: The chosen maximum is provisional until OQ3 is decided, so the measurement at the maximum is a measurement at the placeholder (OQ3) +- risk: 100 lines is the example's promise and no platform limit, so the run at the maximum shows what an order of that size costs on this design and bounds no other command (OQ3, D10) - risk: Reused deciders may carry the two-authority shape of today's platform; each one is checked against the one-authority rule before it is reused (First experiment, D3) - assumption: Nested calls cost more than helpers, by an amount Probe 3 and this experiment measure (F4, Probe 3) - assumption: Transactions have limits, and the maximum is chosen under them (F13) @@ -57,8 +57,7 @@ The six cost targets are the constraint Specs this workflow is constrained by. T ### Open questions - [non-blocking] OQ1: whether a trivial context with no invariants of its own may be plain tables behind lint rules is decided after this experiment measures component call cost (OQ1, Probe 3) -- [non-blocking] OQ3: the largest order `PlaceOrder` supports is a product decision; the experiment measures with a provisional maximum of 100 lines and reports the numbers per line so the owner can choose (OQ3) -- [non-blocking] Extension E-46: the doc asks for one more lifecycle command and one essential summary without naming them; the option taken here is `CancelOrder`, which releases the allocation, and the order summary read model, with 100 as the provisional maximum for measurement (E-46, First experiment) +- [non-blocking] Extension E-46: the doc asks for one more lifecycle command and one essential summary without naming them; the option taken here is `CancelOrder`, which releases the allocation, and the order summary read model (E-46, First experiment) - [non-blocking] Extension E-47: the doc lists what to count but not how; the option taken here is a measurement record per run captured from the disposable backend's function execution log, documents and bytes from the `usageStats` of the command's top-level completion record, and the calls per context from a pure test of the use case's executor, because the log holds no record of a component call inside a mutation, with any count the backend does not expose recorded as a gap; the owner has not ruled (E-47, First experiment) - [non-blocking] Extension E-47, the documents a command reads: the design expected one `PlaceOrder` of N lines to read about N + 6 documents, and the pinned backend counted 3N + 5; the measurement run decides where the other 2N reads come from (E-47, F13) - [non-blocking] Extensions E-8, E-13, E-15 and E-37, as the experiment's setup takes them: stock exists only through `ReceiveStock`, a run seeds its grants and makes the order summary writable with admin access before its first command, the production composition names no service issuer, its `auth.config.ts` reads its issuer from environment variables, and it is the Convex app whose project directory is `example/`; these are options taken here and the owner has not ruled (E-8, E-13, E-15, E-37, E-46) @@ -72,14 +71,16 @@ The six cost targets are the constraint Specs this workflow is constrained by. T - rule: It reuses the pure deciders and state machines where they fit, under the one-authority rule of D3, and leaves the current app's infrastructure behind (First experiment, D3) - rule: It measures orders of 1 line, 10 lines and the chosen maximum, without and with stock contention (First experiment, Sc L2-3) - rule: It counts top-level commits, function and component calls, documents read and written, and rows left behind, with healthy-path and retry costs reported separately (First experiment) -- rule: Semantics come first, speed second; latency and throughput targets are product decisions made before the benchmark (First experiment) +- rule: Semantics come first, speed second; the latency and throughput targets are set before the benchmark, for the pinned local backend on the machine of the run, a target that is missed is reported as not held with its number, and no target is changed after the run (First experiment) - rule: The cost targets are one top-level commit per successful command, zero core projection jobs, O(N) business work allowed, one public command execution per business intent, no application-wide counter, and no queue recovery needed for essential reads (First experiment) - rule: The results feed the open question on component cost and the Layer 3 decisions (First experiment, OQ1, Decision method rule 4) -- rule: Native acceptance runs with production configuration: the same authority, schemas, concurrency and code path as a release (Sc L2-9) +- rule: Native acceptance runs with production configuration, read as the local observation: the production composition on the pinned local backend, with the same authority, schemas, concurrency and code path a release has; no pass of the experiment claims parity with a hosted deployment (Sc L2-9) +- rule: The concurrency half of Sc L2-9 is the contention run on the production composition, and a run on a hosted deployment is deferred together with the function-call quota half of Probe 3 (Sc L2-9, Probe 3) - rule: [extension] The native tier obtains its identities from tokens the harness signs for the fixture issuer that the deployment's environment variables name, carried by ordinary clients, so the actor mapping, the grants, the namespaces, the entry points and Convex's visibility check are the production ones; the issuer is the identity source, the one named difference from production, and every run's evidence record states it (E-13, S13, Sc L2-9) - rule: Every run is recorded as acceptance evidence: commit, installed layers, backend and dependency versions, configuration, dataset, command and result, and a run with adjusted configuration states how it differs from production (Acceptance scenarios) - rule: The kernel's fixture app is separate from the example app, and test-only functions never ship (Acceptance scenarios) -- rule: [extension] The second lifecycle command is `CancelOrder`, the essential summary is the order summary read model, and the provisional maximum is 100 lines (E-46, First experiment) +- rule: [extension] The second lifecycle command is `CancelOrder` and the essential summary is the order summary read model (E-46, First experiment) +- rule: The largest order `PlaceOrder` supports is 100 lines, the bound its declaration carries; it is the example's promise and no platform limit, because a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings, which settles OQ3 (OQ3, D10, F13) - rule: [extension] Each measured run produces one measurement record, and retry cost is the difference between a run with contention and the same run without it (E-47, First experiment) - rule: [extension] Stock exists only through `ReceiveStock`: every run, test and measurement creates its stock items by sending that command, and no function writes a stock item any other way (E-46, Law 1) - rule: [extension] A run seeds its grants by running the production composition's internal grant mutation with admin access before its first command, the read permissions `orders.read` and `inventory.read` among them, as `spec:command.actor-and-scope` pins for a tenant's first grant; the production composition names no service issuer, so every public caller is a human actor (E-37, E-15, D11) @@ -89,9 +90,10 @@ The six cost targets are the constraint Specs this workflow is constrained by. T - Build Layer 2: the `PlaceOrder` use case calling Orders once and Inventory once, the `ReceiveStock` command that creates stock, the `CancelOrder` use case releasing the allocation, the order summary read model written in both, and journal inspection queries (First experiment, D10, D8, E-46) - Build the rebuild of the order summary as a generation, the order allocation history view of `spec:application.orders-inventory-example` with its write-pause rebuild for Sc L2-6 only, and the restore drill on the experiment's dataset (First experiment, D9, D19, Sc L2-6, E-46) - Run every Layer 0, 1 and 2 scenario on a disposable native backend with production configuration and record each run as evidence (First experiment, Sc L2-9, Acceptance scenarios) -- Measure `PlaceOrder` for 1 line, 10 lines and the provisional maximum without contention, one run each, and record a measurement per run (First experiment, Sc L2-3, E-47) +- Measure `PlaceOrder` for 1 line, 10 lines and the maximum of 100 lines without contention, one run each, and record a measurement per run, with the latency of each size taken as `targetLatency` states (First experiment, Sc L2-3, E-47) - Measure the same three sizes with stock contention, 2, 8 and 32 concurrent `PlaceOrder` commands for the same stock item, and record the retry cost separately from the healthy path and the count of commands the engine failed after its bounded optimistic-concurrency retries (First experiment, Sc L1-11, F1, E-47) -- Compare each measurement with the six cost targets and record which hold and which do not, with the per-line numbers (First experiment, Sc L2-3) +- Measure throughput without contention as `targetThroughput` states, 8 callers at once placing ten-line orders on stock items no other caller touches for at least 10 seconds, and record the orders completed per second with the local backend's write cap beside the number (First experiment, E-47) +- Compare each measurement with the six cost targets and the speed targets, and record which hold and which do not, with the per-line numbers (First experiment, Sc L2-3) - Report the cost of one component call from the measurements to OQ1 and Probe 3, and hand the measurements to the Layer 3 decisions (First experiment, OQ1, Probe 3) ## Design @@ -104,14 +106,17 @@ The experiment is an application, a test suite and a report. The application is - captureDocuments: documents and bytes read and written come from the `usageStats` of the command's top-level completion record, `databaseReadDocuments`, `databaseWriteDocuments`, `databaseReadBytes` and `databaseWriteBytes`, which count what the component calls inside the mutation read and wrote; a count the backend does not expose is recorded as `not exposed` (E-47, First experiment, E-15) - captureRowsLeftBehind: rows left behind are the documents a run created that are not state, events, receipts or read-model rows: markers, registry rows, gate and audit records, counted by table (E-47, First experiment, D19) - captureRetries: optimistic-concurrency retries are not observable from inside a mutation; the contention run reports them from the backend's log where exposed, else the retry path is measured as the difference between the contention run and the healthy run (E-47, F1) -- datasetSizes: 1 line, 10 lines and 100 lines as the provisional maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) +- datasetSizes: 1 line, 10 lines and 100 lines, the maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) - contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with the stock set so that the first is applied and the rest are rejected for short stock, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry (First experiment, Sc L1-11, F1, E-5, E-47) -- limitOrderLines: the maximum number of lines per `PlaceOrder` is a product decision, open under OQ3; the experiment measures at 100 and reports documents read and written per line, so the owner can set it under the 1 second timeout and the F13 ceilings (OQ3, F13) +- limitOrderLines: 100 lines per `PlaceOrder`, the largest order the example supports and the bound its declaration carries, chosen under the 1 second timeout, the F13 ceilings and the adapter's stream and byte bounds; it is the example's promise and no platform limit, and the experiment reports documents read and written per line (OQ3, F13) - limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate read, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer (E-47, F13) - streamBudgets: the stock item stream declares `budgetBytes` of 16 KiB and the order stream keeps the default 256 KiB, as `spec:application.orders-inventory-example` pins, so the allocation call's byte bound, 8 MiB over the planned streams' budgets, admits 512 stock streams and the adapter's count bound of `min(maxStreams, 256)` admits the 100 of the maximum order with room, while the single order stream costs at most 256 KiB; at the default budget on both stream types the byte bound would admit only 32 stock streams and the maximum run would be rejected `operationTooLarge`, which is why the budget is pinned here and the run at the maximum measures it (E-2, E-22, E-46, F13, Sc L2-3) -- productDecisions: latency and throughput targets are set by the owner before the benchmark and recorded with the run; the experiment reports latency but does not set a target (First experiment) -- deliverables: the application in this repository, beside the design, which settles OQ4, the evidence records of every scenario run, and the measurement report with the six targets marked held or not held (First experiment, OQ4, Acceptance scenarios) +- productDecisions: the maximum order size and the speed targets of `targetLatency`, `targetThroughput` and `targetContention` are set before the benchmark and recorded with the run; the speed targets hold for the pinned local backend on the machine of the run and say nothing of a hosted deployment; a target that is missed is reported as not held with its number, and no target is changed after the run (First experiment, OQ3) +- targetLatency: without contention, latency is the execution time of the top-level mutation as the backend's function log reports it, taken over at least 30 applied `PlaceOrder` commands of each size after 5 discarded; the median is at most 100 ms at 1 line, 100 ms at 10 lines and 400 ms at 100 lines, and the 95th percentile at most 200 ms, 200 ms and 800 ms (First experiment, E-47) +- targetThroughput: without contention, 8 callers at once, each placing ten-line orders on stock items no other caller touches, complete at least 20 orders a second over at least 10 seconds, with the local backend's write cap of 4 MiB a second stated beside the number (First experiment, E-47) +- targetContention: with N `PlaceOrder` commands at once on one stock item that has stock for one, at N of 2 and 8 every command ends applied or rejected, one applied and the rest rejected for short stock, and the engine fails none after its bounded retries; at N of 32 the same count is reported, and a command the engine failed is a recorded cost and no failure of the experiment, because the caller repeats it under the same request key (First experiment, Sc L1-11, F1) +- deliverables: the application in this repository, beside the design, which settles OQ4, the evidence records of every scenario run, and the measurement report with the six cost targets and the speed targets marked held or not held (First experiment, OQ4, Acceptance scenarios) - authProvider: [extension] the production composition's `auth.config.ts` declares one `customJwt` provider whose `issuer`, `applicationID` and `jwks` are environment variables of the deployment, as `spec:platform.native-harness` pins for a composition that runs natively (E-13, D11) - acceptanceRows: [extension] the rows the experiment's acceptance check requires are Sc L0-1, Sc L0-2, Sc L1-1, Sc L1-2, Sc L1-3, Sc L1-4, Sc L1-5, Sc L1-6, Sc L1-7, Sc L1-8, Sc L1-9, Sc L1-10, Sc L1-11, Sc L1-12, Sc L2-1, Sc L2-2, Sc L2-3, Sc L2-4, Sc L2-5, Sc L2-6, Sc L2-7, Sc L2-8 and Sc L2-9, which are the rows the doc's table gives Layer 0, 1 or 2 (E-17, First experiment, Acceptance scenarios) diff --git a/design/specs/application/parent-use-cases.sdp.md b/design/specs/application/parent-use-cases.sdp.md index 5e94e7b..4270afa 100644 --- a/design/specs/application/parent-use-cases.sdp.md +++ b/design/specs/application/parent-use-cases.sdp.md @@ -44,14 +44,13 @@ The boundary sits at the component API. The use case sees DTOs and stream versio - outcome: Every cross-context rule lives in a parent use case that runs as one top-level mutation with one list-shaped call per context, one receipt and one outcome, so that either the whole operation commits or nothing of it is stored (D1, D10) - value: A maintainer finds every rule that spans two contexts in one place, a new use case needs only its input, its context calls and the read models it maintains, and no saga, command bus or lock protocol sits between local contexts (D10, D1, Thesis) - risk: The parent grows with each cross-context rule, so use cases are grouped by business flow (D10) -- risk: An operation too large for one transaction is rejected or becomes a separate import command with honest partial progress, and the largest order the placement command supports is still a product decision (D10, F13, OQ3) +- risk: An operation too large for one transaction is rejected or becomes a separate import command with honest partial progress, and each use case's declaration carries its own bound, chosen by its author under those limits (D10, F13, OQ3) - assumption: Component calls commit or roll back with the calling mutation, and each call is a sub-transaction isolated from other calls (F2, S2) - assumption: A `ConvexError` thrown inside a component sub-transaction reaches the use case with its data intact, which Probe 2 showed on the pinned backend (F14, Probe 2) - assumption: Transactions have limits, and they are one budget across nested calls and components, as the docs state for nested calls and Probe 4 showed for both (F13, Probe 4) ### Open questions -- [non-blocking] OQ3: the largest order `PlaceOrder` supports is a product decision; the limit bullet below carries a placeholder and the first experiment measures with a provisional maximum (OQ3, D10) - [non-blocking] Extension E-41: the doc says a too-large operation is rejected or becomes a separate import command with honest partial progress; it does not say what the import command records. The option taken here: the input bound is the declaration's `bounds.maxItems` of `spec:command.command-declaration` under E-38, checked by the pipeline's step 1 and by no body, the rejection is the outcome boundary's reserved `operationTooLarge`, and this extension owns only the import command, a caller-driven sequence of bounded batch commands that record progress on an `imports` row (E-41, E-38, D10) - [non-blocking] Extension E-44: the doc says keeping a refused request as a fact is a business policy chosen per command, and the executor signature is pinned by `spec:command.command-declaration`; what neither says is how a use case declares what a context's business failure means to it. The option taken here is a `BusinessFailurePolicy` per context call, carry or reject, applied inside the executor body through the pinned helper `carryOrReject` right after the context call, because the executor is a plain function and the declaration carries no slot for it; `reject` rolls back the events the context recorded as its business failure, which D4 says belong in history, and the doc's per-command choice is read as the use case's because D10 makes the use case the command, so the owner confirms that a context's business failure may be overridden by its caller; the throw shape, the kernel's bare `Rejection` that a context and a use case's own code both throw and the pipeline's rethrow normalizes, is taken here, under E-5 for the boundary's rethrow, and the owner has not ruled on it (E-44, E-5, D4, D7, D10) - [non-blocking] Extension E-39: the doc fixes no order for a use case's context calls; the option taken here is one fixed order the executor states, the creating context first, with no failure handling of a later call, because the calls before it roll back with the mutation; the owner has not ruled (E-39, D1, D10) @@ -113,7 +112,7 @@ A use case is declared with the command declaration of `spec:command.command-dec - indexImportsByImport: `.index("by_import", ["tenantId", "importId"])` (E-41) - indexImportsByImportUse: the batch command loads the import row by tenant and import ID, updates `done` or `failed` in its own transaction, and the caller's progress query reads the same row (E-41, D10) - limitCallsPerContext: 1 component call per context per use case, counted by the first experiment (D10, Sc L2-3) -- limitOrderLines: the maximum number of lines per `PlaceOrder` is a product decision, written `MAX_ORDER_LINES` and open under OQ3; it is chosen so one order reads and writes about three documents per line plus a constant, far below the 32,000 scanned and 16,000 written per transaction and inside the 1 second budget (F13, OQ3) +- limitOrderLines: 100 lines per `PlaceOrder`, the largest order the example supports and the bound its declaration carries, the example's promise and no platform limit; it is chosen so one order reads and writes about three documents per line plus a constant, far below the 32,000 scanned and 16,000 written per transaction and inside the 1 second budget (F13, OQ3) - limitInputBound: every use case's input bound is its declaration's `bounds.maxItems`, checked at the pipeline's step 1, a ceiling below which the experiment measures its documents read and written; each context's adapter keeps its own `maxStreams` and byte bound and the smallest bites first (E-38, F13, D19) - limitImportBatch: `[extension]` an import batch carries at most its own declaration's `bounds.maxItems` items and is one transaction with one receipt, so an import of N items is at least N divided by that bound commands (E-41, E-38, D10) diff --git a/design/specs/command/command-pipeline.sdp.md b/design/specs/command/command-pipeline.sdp.md index c7ef51b..1ea95dd 100644 --- a/design/specs/command/command-pipeline.sdp.md +++ b/design/specs/command/command-pipeline.sdp.md @@ -124,7 +124,7 @@ The pipeline depends on three contracts it does not own: the context component's - admissionHook: `admission?: (ctx: MutationCtx, call: PipelineCall) => Promise<{ admitted: true } | { admitted: false; code: "rateLimited" | "capacity"; retryAfterMs?: number }>` on the declaration, absent by default; a refusal throws the transient error and any consumption it wrote rolls back with the throw, as it does with every later throw of the same mutation, so an admitted call that is then rejected leaves no consumption either (D6, D4, E-32) - subTransactionLimits: `ctx.runMutation` accepts `{ transactionLimits }` per call; the limits are shared across the boundary, as Probe 4 showed; the design passes no `transactionLimits` on a context call, and whether a call should reserve room for the work after it is an open question (S3, Probe 4, F13) - limitMutationTimeout: the whole pipeline runs under the 1-second query and mutation timeout; every per-command bound in a declaration is chosen so that one command's reads and writes finish well inside it (F13) -- limitTransaction: 16 MiB read, 16 MiB written, 32,000 documents scanned, 4,096 index ranges read, 16,000 documents written per transaction; the per-command `bounds` in the declaration, such as the maximum order lines, are ceilings chosen against these, and the largest order is a product decision (F13, D10, OQ3) +- limitTransaction: 16 MiB read, 16 MiB written, 32,000 documents scanned, 4,096 index ranges read, 16,000 documents written per transaction; the per-command `bounds` in the declaration, such as the maximum order lines, are ceilings each command's author chooses against these, and 100 lines, the largest order the example's `PlaceOrder` supports, is that command's bound and no platform limit (F13, D10, OQ3) - limitContextCalls: one `ctx.runMutation` per context named by the declaration per command; O(N) business work inside a call is fine, O(N) calls is not (D10) - limitTooLarge: an input that exceeds the declaration's `bounds`, `maxItems` items or `maxBytes` bytes measured with `getConvexSize`, throws rejection `operationTooLarge` at step 1, the one checkpoint for the input bound in the corpus; a context's adapter keeps its own `maxStreams` and byte bound and the smaller bites first; the pipeline never splits an input, because splitting silently changes the contract (D10, D4, E-38) - fnUtf8Length: [extension] `utf8Length(value: string): number` returns the number of bytes `TextEncoder` produces for the string, so an unpaired surrogate counts as the three bytes of U+FFFD; for a well-formed string this equals `getConvexSize(value) - 2`, and for a string with an unpaired surrogate it does not; every text bound of step 1 and of the persistence adapter's step 1 is measured with it, and the command library and the context library both export it (F13, E-34) diff --git a/design/specs/context/batch-shaped-api.sdp.md b/design/specs/context/batch-shaped-api.sdp.md index 8b6ccb5..cb9c931 100644 --- a/design/specs/context/batch-shaped-api.sdp.md +++ b/design/specs/context/batch-shaped-api.sdp.md @@ -21,10 +21,6 @@ Context APIs take lists, such as `inventory.allocate({ lines })`, so a use case - risk: The transaction limits are one budget across nested calls and components, as Probe 4 showed, so every bound here is checked against the sum (F13, Probe 4) - assumption: Transactions have limits, and platform limits are ceilings, not batch sizes (F13, D19) -### Open questions - -- [non-blocking] OQ3: the largest order the placement command supports is a product decision; each operation's `maxStreams` is set from it and the default ceiling of 256 stands until then (OQ3, D10) - ## Rule - Context APIs take lists, so a use case makes one call per context rather than one per line (D10) @@ -46,7 +42,7 @@ Context APIs take lists, such as `inventory.allocate({ lines })`, so a use case - exampleAllocate: `inventory.allocate({ orderId, lines: [{ stockItemId, quantity }] })` plans one stream command `{ commandType: "allocate", orderId, quantity }` per distinct stock item, with the quantities of its lines summed, and returns `{ lines: [{ stockItemId, quantity }] }`, one entry per stock item with the quantity allocated, with the stream versions on the outcome (D10, Sc L2-3) - limitStreamsPerCall: `maxStreams` per operation, ceiling 256, and 8 MiB of stream documents at the planned stream types' `budgetBytes`, so the allocation operation's 100 stock streams fit only because the stock stream type declares a small budget, 16 KiB as `spec:application.orders-inventory-example` pins, under which the byte bound admits 512 and the count bound the full 256 (E-22, E-2, E-46, F13) - limitDocumentsWrittenPerCall: 800 documents and 8 MiB per call (E-22, F13) -- limitOrderLines: the maximum lines per `PlaceOrder` is a product decision and a placeholder in the first experiment; `maxStreams` of the allocation operation follows it (OQ3, D10, F13) +- limitOrderLines: `PlaceOrder`'s declaration carries its own bound, 100 lines, the largest order the example supports, chosen under the stream and byte ceilings above and no platform limit; `maxStreams` of the allocation operation follows it, under the ceiling of 256 (OQ3, D10, F13) - importCommand: a list above the bound is served by a separate import command that records honest partial progress per batch in its own stream, never by the operation splitting silently (D10) ## Verification — reviewed diff --git a/design/specs/decisions/d03-events-only-source-of-next-state.sdp.md b/design/specs/decisions/d03-events-only-source-of-next-state.sdp.md index 3561cfa..a183198 100644 --- a/design/specs/decisions/d03-events-only-source-of-next-state.sdp.md +++ b/design/specs/decisions/d03-events-only-source-of-next-state.sdp.md @@ -24,7 +24,7 @@ Provenance: carried from v0.1; the two gaps at the end are new. Feature · Trace ### Open questions - [non-blocking] Gap, initial state: the v0.1 interface has none; add `initial()` or let `evolve` accept an empty state so a rebuild from the creation event has a defined start; ruled provisionally as E-1 in the kernel's initial-state decision (D3, E-1) -- [non-blocking] Gap, state across several documents: prefer one document per stream while it fits the size budget, and where it cannot, derive the document writes from the folded state in one place per context; ruled provisionally as E-2 in the kernel's state-document mapping decision, with the size budget open under OQ3 (D3, E-2, OQ3) +- [non-blocking] Gap, state across several documents: prefer one document per stream while it fits the size budget, and where it cannot, derive the document writes from the folded state in one place per context; ruled provisionally as E-2 in the kernel's state-document mapping decision, where one document per stream stays the default and the size budget is a provisional number of E-2 (D3, E-2, OQ3) ## Decision diff --git a/design/specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md b/design/specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md index 86b45a0..9adb645 100644 --- a/design/specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md +++ b/design/specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md @@ -23,13 +23,9 @@ Contexts never read each other's tables and never call each other. A rule linkin - outcome: Contexts never read each other's tables and never call each other; every cross-context rule lives in a parent use case in one transaction or in an obligation, and a use case makes one list-shaped call per context (D10) - value: Every cross-context rule is visible in one place, and O(N) business work stays O(1) in component calls (D10) - risk: The parent grows with each cross-context rule, so use cases are grouped by business flow (D10) -- risk: An operation too large for one transaction is rejected or becomes a separate import command with honest partial progress, and the maximum order size is a product decision still open (D10, F13, OQ3) +- risk: An operation too large for one transaction is rejected or becomes a separate import command with honest partial progress, and each command's declaration carries its own bound, chosen by its author under those limits (D10, F13, OQ3) - assumption: Transactions have limits (F13) -### Open questions - -- [non-blocking] OQ3: the largest order the placement command supports is a product decision; until it is made the limit is a placeholder in the first experiment (OQ3, D10) - ## Decision - context: The concern is where a rule that spans two contexts lives and how many calls a use case makes; Convex gives component isolation, where a component cannot read data not explicitly provided to it, and per-transaction limits that bound one use case (D10, F2, F13) @@ -45,5 +41,5 @@ Contexts never read each other's tables and never call each other. A rule linkin - rationale: O(N) business work is fine; O(N) component calls or orchestration is not the default (D10) - consequence: The parent grows with each cross-context rule, so use cases are grouped by business flow (D10) - consequence: A context API takes lists, such as `inventory.allocate({ lines })`, so a use case makes one call per context rather than one per line (D10) -- consequence: The first experiment counts one call per context per use case and measures orders of 1 line, 10 lines and the chosen maximum (D10, First experiment, Sc L2-3) +- consequence: The first experiment counts one call per context per use case and measures orders of 1 line, 10 lines and the maximum of 100 lines, the bound `PlaceOrder`'s declaration carries, which is the example's promise and no platform limit (D10, First experiment, Sc L2-3, OQ3) - consequence: The standing cost is none in mechanism; the cost is the discipline of list-shaped context APIs and of a parent that holds every cross-context rule (D10) diff --git a/design/specs/kernel/state-document-mapping.sdp.md b/design/specs/kernel/state-document-mapping.sdp.md index be1fc1e..c1ec22d 100644 --- a/design/specs/kernel/state-document-mapping.sdp.md +++ b/design/specs/kernel/state-document-mapping.sdp.md @@ -12,20 +12,19 @@ relations: Provenance: new; the doc names the gap, states the preference and leaves the budget open. Story · Traces: D2, D3, D5, D10, D18, F13, OQ3, Sc L2-3, E-2, E-3, E-25. -Folding is simple when a stream's state is one document. When state spans several documents, such as an order with separate line rows or stock spread over rows, the adapter needs a mapping from folded state to documents, and if each command writes that mapping by hand a second source of state creeps back. This decision keeps one document per stream as the default while the state fits a size budget, and where it cannot, derives every document write from the folded state in one mapping per stream type. The budget, `budgetBytes`, is one number per stream type and means the whole stream: the one document of the single mapping, or the head and every part of the derived mapping together. Every bound in the corpus that multiplies stream documents multiplies that number. The budget is a design number under OQ3; the maximum order size that the placement command supports is the product decision that tests it. +Folding is simple when a stream's state is one document. When state spans several documents, such as an order with separate line rows or stock spread over rows, the adapter needs a mapping from folded state to documents, and if each command writes that mapping by hand a second source of state creeps back. This decision keeps one document per stream as the default while the state fits a size budget, and where it cannot, derives every document write from the folded state in one mapping per stream type. The budget, `budgetBytes`, is one number per stream type and means the whole stream: the one document of the single mapping, or the head and every part of the derived mapping together. Every bound in the corpus that multiplies stream documents multiplies that number. The budget is a design number. A command's declaration carries its own largest input, chosen by its author under the adapter's stream and byte ceilings that the budget sets, so the 100 lines of the example's `PlaceOrder` are that command's promise and no platform limit. ## Intent - outcome: The saved representation of a stream's state is derived from the folded state by one mapping per stream type, and the default mapping is one document that holds the whole state and the stream metadata (D3, E-2) - value: No command writes documents by hand, so the fold stays the one authority for state even when state spans documents (D3) - risk: A state that outgrows the budget forces the derived mapping, which multiplies the documents one command reads and writes; the first experiment measures documents read and written per order size (D10, First experiment, Sc L2-3) -- risk: The document budget below is a design number; if OQ3 sets a maximum order that does not fit it, the budget or the mapping changes (OQ3, E-2) +- risk: The document budget below is a design number; a command whose declared bound does not fit it needs a larger budget or the derived mapping (OQ3, E-2) - assumption: A document holds at most 1 MiB and a transaction reads at most 16 MiB and 32,000 documents (F13) ### Open questions -- [non-blocking] Extension E-2: the doc prefers one document per stream within a size budget and names no budget; this Spec sets `budgetBytes` as one number per stream type meaning the whole stream, the single document or the head and every part together, at most 256 KiB for the single mapping and at most 512 KiB over at most 32 parts for the derived mapping, provisionally, lets a stream type declare a smaller `budgetBytes` on its mapping, caps the derived budget at half the document ceiling because a baseline event holds the whole migrated state in one document, and derives every bound on streams per call, list page, backfill batch and baseline batch from that budget so that the 16 MiB read ceiling binds in bytes and not only in documents; the owner confirms the numbers with OQ3 (D3, D5, OQ3, F13, E-2, E-25) -- [non-blocking] OQ3: one document per stream as the default, and the largest order the placement command supports, are product decisions; the design assumes one document and leaves the maximum lines as a placeholder in the first experiment (OQ3, D10) +- [non-blocking] Extension E-2: the doc prefers one document per stream within a size budget and names no budget; this Spec sets `budgetBytes` as one number per stream type meaning the whole stream, the single document or the head and every part together, at most 256 KiB for the single mapping and at most 512 KiB over at most 32 parts for the derived mapping, provisionally, lets a stream type declare a smaller `budgetBytes` on its mapping, caps the derived budget at half the document ceiling because a baseline event holds the whole migrated state in one document, and derives every bound on streams per call, list page, backfill batch and baseline batch from that budget so that the 16 MiB read ceiling binds in bytes and not only in documents; the owner confirms the numbers (D3, D5, OQ3, F13, E-2, E-25) ## Decision @@ -40,6 +39,7 @@ Folding is simple when a stream's state is one document. When state spans severa - consequence: The stream row carries the metadata that enumeration and deletion need, including the deleted-subject marker, in both mappings (D2, E-3) - consequence: A context switching a stream type from `single` to `derived` rewrites its stream rows through the new mapping in bounded batches; no event changes and `stateSchemaVersion` does not move, so it is a change of representation, not of meaning (D5, E-25) - consequence: The derived mapping writes only the parts whose value changed, compared against the loaded parts, so a command on a large state costs the parts it touched plus the head (E-2) +- consequence: One document per stream stays the default, with the derived mapping for state that outgrows its budget, and the largest order the placement command supports is the bound its declaration carries, 100 lines for the example's `PlaceOrder`, chosen under the adapter's stream and byte ceilings and no platform limit, which settles OQ3 (OQ3, D10, E-2) - consequence: The standing cost is one mapping declaration per stream type and, for the derived mapping, one extra table, one index and one extra read per part on every load (E-2, E-3) ## Design diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 551f56a..88848bb 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -10,7 +10,7 @@ relations: Feature · Detail: extension · Traces: Acceptance scenarios (tiers, evidence), S13, Sc L2-9, D11, E-13, E-15. -The doc says that native tests prove component and nested-mutation behavior, scheduling, contention and deployment, that the kernel's fixture app is separate from the example app, that test-only functions never ship and that every test owns its disposable backend. It does not say how a test obtains an identity, what the fixture app contains, how a fault enters a run, what a test may read or what it controls. This Spec carries those rules as two extensions, E-13 and E-15. Every rule here is the corpus's provisional reading until the owner rules. The doc's own rules on tiers and evidence stay in `spec:platform.acceptance-contract`. +The doc says that native tests prove component and nested-mutation behavior, scheduling, contention and deployment, that the kernel's fixture app is separate from the example app, that test-only functions never ship and that every test owns its disposable backend. It does not say how a test obtains an identity, what the fixture app contains, how a fault enters a run, what a test may read or what it controls. This Spec carries those rules as two extensions, E-13 and E-15. Every rule here is the corpus's provisional reading until the owner rules, except the reading of Sc L2-9 in the first rule, which is settled. The doc's own rules on tiers and evidence stay in `spec:platform.acceptance-contract`. ## Intent @@ -27,7 +27,7 @@ The doc says that native tests prove component and nested-mutation behavior, sch ## Rule -- [extension] Parity with a release for Sc L2-9 means the same code path, schema, concurrency and authority model: the grants, namespaces, entry points and actor mapping, and Convex's own visibility check on every call a caller under test makes (E-13, S13, Sc L2-9) +- [extension] Sc L2-9 is read as the local observation: the production composition on the pinned local backend, with the same code path, schema, concurrency and authority model a release has, the authority model being the grants, namespaces, entry points and actor mapping, and Convex's own visibility check on every call a caller under test makes; no native pass claims parity with a hosted deployment (E-13, S13, Sc L2-9) - [extension] A caller under test is an ordinary client that carries a token signed by the fixture issuer, or no token (E-13) - [extension] A composition that runs natively reads its issuer, its application ID and its key set from the deployment's environment variables, and the harness sets the key set to a data URI that holds the fixture issuer's public key (E-13, S13) - [extension] No caller under test carries the admin key, alone or acting as an identity, because a call made with it passes Convex's visibility check (E-13) From 17f9972b1b73b5bd53ef1cdc7158f6763a150985 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:00:19 +0200 Subject: [PATCH 003/155] design: the check's corrections to the largest order, the speed targets and the reading of Sc L2-9 The latency sample counts operations, the speed targets are set for the local backend rather than said to hold, the contention shape admits engine failures, the mapping risk and the bound's ceilings say no more than the ruling, and the multi-document gap keeps only its size budget open. Co-Authored-By: Claude Opus 5.5 --- design/specs/application/first-experiment.sdp.md | 6 +++--- design/specs/application/parent-use-cases.sdp.md | 2 +- .../decisions/d03-events-only-source-of-next-state.sdp.md | 4 ++-- .../decisions/d10-contexts-meet-in-parent-use-cases.sdp.md | 2 +- design/specs/kernel/state-document-mapping.sdp.md | 4 ++-- design/specs/platform/native-harness.sdp.md | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 23a6f07..a745cb6 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -107,13 +107,13 @@ The experiment is an application, a test suite and a report. The application is - captureRowsLeftBehind: rows left behind are the documents a run created that are not state, events, receipts or read-model rows: markers, registry rows, gate and audit records, counted by table (E-47, First experiment, D19) - captureRetries: optimistic-concurrency retries are not observable from inside a mutation; the contention run reports them from the backend's log where exposed, else the retry path is measured as the difference between the contention run and the healthy run (E-47, F1) - datasetSizes: 1 line, 10 lines and 100 lines, the maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) -- contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with the stock set so that the first is applied and the rest are rejected for short stock, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry (First experiment, Sc L1-11, F1, E-5, E-47) +- contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with stock for one, so that one is applied and the rest are rejected for short stock or failed by the engine after its bounded retries, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry (First experiment, Sc L1-11, F1, E-5, E-47) - limitOrderLines: 100 lines per `PlaceOrder`, the largest order the example supports and the bound its declaration carries, chosen under the 1 second timeout, the F13 ceilings and the adapter's stream and byte bounds; it is the example's promise and no platform limit, and the experiment reports documents read and written per line (OQ3, F13) - limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate read, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer (E-47, F13) - streamBudgets: the stock item stream declares `budgetBytes` of 16 KiB and the order stream keeps the default 256 KiB, as `spec:application.orders-inventory-example` pins, so the allocation call's byte bound, 8 MiB over the planned streams' budgets, admits 512 stock streams and the adapter's count bound of `min(maxStreams, 256)` admits the 100 of the maximum order with room, while the single order stream costs at most 256 KiB; at the default budget on both stream types the byte bound would admit only 32 stock streams and the maximum run would be rejected `operationTooLarge`, which is why the budget is pinned here and the run at the maximum measures it (E-2, E-22, E-46, F13, Sc L2-3) -- productDecisions: the maximum order size and the speed targets of `targetLatency`, `targetThroughput` and `targetContention` are set before the benchmark and recorded with the run; the speed targets hold for the pinned local backend on the machine of the run and say nothing of a hosted deployment; a target that is missed is reported as not held with its number, and no target is changed after the run (First experiment, OQ3) -- targetLatency: without contention, latency is the execution time of the top-level mutation as the backend's function log reports it, taken over at least 30 applied `PlaceOrder` commands of each size after 5 discarded; the median is at most 100 ms at 1 line, 100 ms at 10 lines and 400 ms at 100 lines, and the 95th percentile at most 200 ms, 200 ms and 800 ms (First experiment, E-47) +- productDecisions: the maximum order size and the speed targets of `targetLatency`, `targetThroughput` and `targetContention` are set before the benchmark and recorded with the run; the speed targets are set for the pinned local backend on the machine of the run and say nothing of a hosted deployment; a target that is missed is reported as not held with its number, and no target is changed after the run (First experiment, OQ3) +- targetLatency: without contention, latency is the execution time of the top-level mutation as the backend's function log reports it, taken over at least 30 `PlaceOrder` operations of each size after 5 discarded; the median is at most 100 ms at 1 line, 100 ms at 10 lines and 400 ms at 100 lines, and the 95th percentile at most 200 ms, 200 ms and 800 ms (First experiment, E-47) - targetThroughput: without contention, 8 callers at once, each placing ten-line orders on stock items no other caller touches, complete at least 20 orders a second over at least 10 seconds, with the local backend's write cap of 4 MiB a second stated beside the number (First experiment, E-47) - targetContention: with N `PlaceOrder` commands at once on one stock item that has stock for one, at N of 2 and 8 every command ends applied or rejected, one applied and the rest rejected for short stock, and the engine fails none after its bounded retries; at N of 32 the same count is reported, and a command the engine failed is a recorded cost and no failure of the experiment, because the caller repeats it under the same request key (First experiment, Sc L1-11, F1) - deliverables: the application in this repository, beside the design, which settles OQ4, the evidence records of every scenario run, and the measurement report with the six cost targets and the speed targets marked held or not held (First experiment, OQ4, Acceptance scenarios) diff --git a/design/specs/application/parent-use-cases.sdp.md b/design/specs/application/parent-use-cases.sdp.md index 4270afa..2532bbd 100644 --- a/design/specs/application/parent-use-cases.sdp.md +++ b/design/specs/application/parent-use-cases.sdp.md @@ -44,7 +44,7 @@ The boundary sits at the component API. The use case sees DTOs and stream versio - outcome: Every cross-context rule lives in a parent use case that runs as one top-level mutation with one list-shaped call per context, one receipt and one outcome, so that either the whole operation commits or nothing of it is stored (D1, D10) - value: A maintainer finds every rule that spans two contexts in one place, a new use case needs only its input, its context calls and the read models it maintains, and no saga, command bus or lock protocol sits between local contexts (D10, D1, Thesis) - risk: The parent grows with each cross-context rule, so use cases are grouped by business flow (D10) -- risk: An operation too large for one transaction is rejected or becomes a separate import command with honest partial progress, and each use case's declaration carries its own bound, chosen by its author under those limits (D10, F13, OQ3) +- risk: An operation too large for one transaction is rejected or becomes a separate import command with honest partial progress, and each command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings (D10, F13, OQ3) - assumption: Component calls commit or roll back with the calling mutation, and each call is a sub-transaction isolated from other calls (F2, S2) - assumption: A `ConvexError` thrown inside a component sub-transaction reaches the use case with its data intact, which Probe 2 showed on the pinned backend (F14, Probe 2) - assumption: Transactions have limits, and they are one budget across nested calls and components, as the docs state for nested calls and Probe 4 showed for both (F13, Probe 4) diff --git a/design/specs/decisions/d03-events-only-source-of-next-state.sdp.md b/design/specs/decisions/d03-events-only-source-of-next-state.sdp.md index a183198..8dec1ef 100644 --- a/design/specs/decisions/d03-events-only-source-of-next-state.sdp.md +++ b/design/specs/decisions/d03-events-only-source-of-next-state.sdp.md @@ -11,7 +11,7 @@ relations: Provenance: carried from v0.1; the two gaps at the end are new. Feature · Traces: D3, Law 2, Law 3, Law 10, Sc L0-1, Sc L0-2, E-1, E-2, OQ3. -`decide(state, command, context)` returns events and a result, or a rejection. The persistence adapter computes the next state as `fold(evolve, state, newEvents)` and saves it. The decider never returns a separate state patch. Commands load current state and apply only the new events; they never replay history. Today's platform has two authorities and nothing keeps them in step; one authority makes state and history agree by construction. The decision depends on D2 because the fold and the save happen inside the context that owns the state. The two gaps the doc names, initial state and state across several documents, are ruled provisionally by Package B as extensions E-1 and E-2. +`decide(state, command, context)` returns events and a result, or a rejection. The persistence adapter computes the next state as `fold(evolve, state, newEvents)` and saves it. The decider never returns a separate state patch. Commands load current state and apply only the new events; they never replay history. Today's platform has two authorities and nothing keeps them in step; one authority makes state and history agree by construction. The decision depends on D2 because the fold and the save happen inside the context that owns the state. The two gaps the doc names, initial state and state across several documents, are ruled provisionally by Package B as extensions E-1 and E-2; of the second, one document per stream as the default, with the derived mapping for state that outgrows its budget, is settled, and the size budget stays provisional. ## Intent @@ -24,7 +24,7 @@ Provenance: carried from v0.1; the two gaps at the end are new. Feature · Trace ### Open questions - [non-blocking] Gap, initial state: the v0.1 interface has none; add `initial()` or let `evolve` accept an empty state so a rebuild from the creation event has a defined start; ruled provisionally as E-1 in the kernel's initial-state decision (D3, E-1) -- [non-blocking] Gap, state across several documents: prefer one document per stream while it fits the size budget, and where it cannot, derive the document writes from the folded state in one place per context; ruled provisionally as E-2 in the kernel's state-document mapping decision, where one document per stream stays the default and the size budget is a provisional number of E-2 (D3, E-2, OQ3) +- [non-blocking] Gap, state across several documents, its size budget: one document per stream stays the default while it fits the size budget, and where it cannot, the document writes are derived from the folded state in one place per context, as the kernel's state-document mapping decision states; the size budget there is a provisional number of E-2 (D3, E-2, OQ3) ## Decision diff --git a/design/specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md b/design/specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md index 9adb645..4d323be 100644 --- a/design/specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md +++ b/design/specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md @@ -23,7 +23,7 @@ Contexts never read each other's tables and never call each other. A rule linkin - outcome: Contexts never read each other's tables and never call each other; every cross-context rule lives in a parent use case in one transaction or in an obligation, and a use case makes one list-shaped call per context (D10) - value: Every cross-context rule is visible in one place, and O(N) business work stays O(1) in component calls (D10) - risk: The parent grows with each cross-context rule, so use cases are grouped by business flow (D10) -- risk: An operation too large for one transaction is rejected or becomes a separate import command with honest partial progress, and each command's declaration carries its own bound, chosen by its author under those limits (D10, F13, OQ3) +- risk: An operation too large for one transaction is rejected or becomes a separate import command with honest partial progress, and each command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings (D10, F13, OQ3) - assumption: Transactions have limits (F13) ## Decision diff --git a/design/specs/kernel/state-document-mapping.sdp.md b/design/specs/kernel/state-document-mapping.sdp.md index c1ec22d..9fdb5a8 100644 --- a/design/specs/kernel/state-document-mapping.sdp.md +++ b/design/specs/kernel/state-document-mapping.sdp.md @@ -12,14 +12,14 @@ relations: Provenance: new; the doc names the gap, states the preference and leaves the budget open. Story · Traces: D2, D3, D5, D10, D18, F13, OQ3, Sc L2-3, E-2, E-3, E-25. -Folding is simple when a stream's state is one document. When state spans several documents, such as an order with separate line rows or stock spread over rows, the adapter needs a mapping from folded state to documents, and if each command writes that mapping by hand a second source of state creeps back. This decision keeps one document per stream as the default while the state fits a size budget, and where it cannot, derives every document write from the folded state in one mapping per stream type. The budget, `budgetBytes`, is one number per stream type and means the whole stream: the one document of the single mapping, or the head and every part of the derived mapping together. Every bound in the corpus that multiplies stream documents multiplies that number. The budget is a design number. A command's declaration carries its own largest input, chosen by its author under the adapter's stream and byte ceilings that the budget sets, so the 100 lines of the example's `PlaceOrder` are that command's promise and no platform limit. +Folding is simple when a stream's state is one document. When state spans several documents, such as an order with separate line rows or stock spread over rows, the adapter needs a mapping from folded state to documents, and if each command writes that mapping by hand a second source of state creeps back. This decision keeps one document per stream as the default while the state fits a size budget, and where it cannot, derives every document write from the folded state in one mapping per stream type. The budget, `budgetBytes`, is one number per stream type and means the whole stream: the one document of the single mapping, or the head and every part of the derived mapping together. Every bound in the corpus that multiplies stream documents multiplies that number. The budget is a design number. A command's declaration carries its own largest input, chosen by its author under the adapter's stream and byte ceilings, so the 100 lines of the example's `PlaceOrder` are that command's promise and no platform limit. ## Intent - outcome: The saved representation of a stream's state is derived from the folded state by one mapping per stream type, and the default mapping is one document that holds the whole state and the stream metadata (D3, E-2) - value: No command writes documents by hand, so the fold stays the one authority for state even when state spans documents (D3) - risk: A state that outgrows the budget forces the derived mapping, which multiplies the documents one command reads and writes; the first experiment measures documents read and written per order size (D10, First experiment, Sc L2-3) -- risk: The document budget below is a design number; a command whose declared bound does not fit it needs a larger budget or the derived mapping (OQ3, E-2) +- risk: The document budget below is a design number; a command bound whose state does not fit it changes the budget or the mapping (OQ3, E-2) - assumption: A document holds at most 1 MiB and a transaction reads at most 16 MiB and 32,000 documents (F13) ### Open questions diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 88848bb..e1da46a 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -10,7 +10,7 @@ relations: Feature · Detail: extension · Traces: Acceptance scenarios (tiers, evidence), S13, Sc L2-9, D11, E-13, E-15. -The doc says that native tests prove component and nested-mutation behavior, scheduling, contention and deployment, that the kernel's fixture app is separate from the example app, that test-only functions never ship and that every test owns its disposable backend. It does not say how a test obtains an identity, what the fixture app contains, how a fault enters a run, what a test may read or what it controls. This Spec carries those rules as two extensions, E-13 and E-15. Every rule here is the corpus's provisional reading until the owner rules, except the reading of Sc L2-9 in the first rule, which is settled. The doc's own rules on tiers and evidence stay in `spec:platform.acceptance-contract`. +The doc says that native tests prove component and nested-mutation behavior, scheduling, contention and deployment, that the kernel's fixture app is separate from the example app, that test-only functions never ship and that every test owns its disposable backend. It does not say how a test obtains an identity, what the fixture app contains, how a fault enters a run, what a test may read or what it controls. This Spec carries those rules as two extensions, E-13 and E-15. Every rule here is the corpus's provisional reading until the owner rules, except the local reading of Sc L2-9 in the first rule, which is settled. The doc's own rules on tiers and evidence stay in `spec:platform.acceptance-contract`. ## Intent From 3aea6151550481514ec1679c24b4b9dd05fa61a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:01:43 +0200 Subject: [PATCH 004/155] design: the mapping's risk names no remedy the ruling does not Co-Authored-By: Claude Opus 5.5 --- design/specs/kernel/state-document-mapping.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/kernel/state-document-mapping.sdp.md b/design/specs/kernel/state-document-mapping.sdp.md index 9fdb5a8..2c998ff 100644 --- a/design/specs/kernel/state-document-mapping.sdp.md +++ b/design/specs/kernel/state-document-mapping.sdp.md @@ -19,7 +19,7 @@ Folding is simple when a stream's state is one document. When state spans severa - outcome: The saved representation of a stream's state is derived from the folded state by one mapping per stream type, and the default mapping is one document that holds the whole state and the stream metadata (D3, E-2) - value: No command writes documents by hand, so the fold stays the one authority for state even when state spans documents (D3) - risk: A state that outgrows the budget forces the derived mapping, which multiplies the documents one command reads and writes; the first experiment measures documents read and written per order size (D10, First experiment, Sc L2-3) -- risk: The document budget below is a design number; a command bound whose state does not fit it changes the budget or the mapping (OQ3, E-2) +- risk: The document budget below is a design number, and a command's declared bound holds only while the stream state that bound produces fits it (OQ3, E-2) - assumption: A document holds at most 1 MiB and a transaction reads at most 16 MiB and 32,000 documents (F13) ### Open questions From 6c8f63b1d49781439e4e0e3d849cad9c049899d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:02:56 +0200 Subject: [PATCH 005/155] test: measure PlaceOrder cost and native contention --- README.md | 2 + design/README.md | 4 +- .../first-experiment.native-contention.sdp.md | 36 ++ .../specs/application/first-experiment.sdp.md | 27 +- package.json | 3 +- scripts/measure-place-order.mjs | 32 ++ tests/native/place-order-contention.test.ts | 65 +++ tests/native/place-order-contention.ts | 100 ++++ tests/native/place-order-measurement.test.ts | 252 ++++++++++ tests/native/place-order-measurement.ts | 457 ++++++++++++++++++ tests/native/place-order-timing.test.ts | 251 ++++++++++ 11 files changed, 1218 insertions(+), 11 deletions(-) create mode 100644 design/specs/application/first-experiment.native-contention.sdp.md create mode 100644 scripts/measure-place-order.mjs create mode 100644 tests/native/place-order-contention.test.ts create mode 100644 tests/native/place-order-contention.ts create mode 100644 tests/native/place-order-measurement.test.ts create mode 100644 tests/native/place-order-measurement.ts create mode 100644 tests/native/place-order-timing.test.ts diff --git a/README.md b/README.md index d451fbf..36fa1d0 100644 --- a/README.md +++ b/README.md @@ -55,3 +55,5 @@ The harness runs every Convex CLI call in the project directory of the compositi `npm run dev` starts one disposable backend and runs `convex dev` against it for the fixture composition; `npm run dev -- production` does the same for the production composition. Each time you save a file of that composition, it rewrites the composition's `_generated` directories. It runs until you press Ctrl-C, then stops the backend and removes it. Commit the generated files as with `npm run codegen`: CI still fails when they are stale. It refuses to start when a `.env.local` file exists in the composition's project directory, the repository root or `example/`, because `convex dev` would write the backend's URL into it. Every native run writes a record to `evidence/runs/`. [`evidence/README.md`](evidence/README.md) says what a record holds and where the record of a push is kept. + +The native measurement files are [the cost cells](tests/native/place-order-measurement.test.ts), [the contention matrix](tests/native/place-order-contention.test.ts) and [the timing cells](tests/native/place-order-timing.test.ts). [The timing command](scripts/measure-place-order.mjs), `npm run measure:place-order`, writes records and prints speed comparisons marked as taken under load. On a quiet machine use `npm run measure:place-order -- --quiet`. The timing cells are skipped by ordinary native test runs. diff --git a/design/README.md b/design/README.md index 96361e1..d0eb65d 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -202 specs · 5 packs · 101 anchors → 308 nodes · 1097 edges (0 errors, 0 warnings) +203 specs · 5 packs · 104 anchors → 312 nodes · 1102 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -154,7 +154,7 @@ Every row of the doc's acceptance table is an example Spec that `refines` and `v | L2-6 | Rebuild a cross-stream history view under a write pause; interrupt | [`spec:application.rebuild`](specs/application/rebuild.sdp.md) | [`.write-pause-rebuild-interrupt`](specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md), [`.write-pause-rebuild-abort`](specs/application/rebuild.write-pause-rebuild-abort.sdp.md) | native | | L2-7 | Rebuild a stream that has a baseline event | [`spec:context.journal`](specs/context/journal.sdp.md) | [`.rebuild-from-baseline`](specs/context/journal.rebuild-from-baseline.sdp.md) | native | | L2-8 | Restore a representative dataset with matching code and configuration | [`spec:application.restore`](specs/application/restore.sdp.md) | [`.restore-representative-dataset`](specs/application/restore.restore-representative-dataset.sdp.md) | end to end | -| L2-9 | Run native acceptance with production configuration | [`spec:application.first-experiment`](specs/application/first-experiment.sdp.md) | [`.native-acceptance-production-configuration`](specs/application/first-experiment.native-acceptance-production-configuration.sdp.md) | end to end | +| L2-9 | Run native acceptance with production configuration | [`spec:application.first-experiment`](specs/application/first-experiment.sdp.md) | [`.native-acceptance-production-configuration`](specs/application/first-experiment.native-acceptance-production-configuration.sdp.md), [`.native-contention`](specs/application/first-experiment.native-contention.sdp.md) | end to end; native | | L3-1 | Duplicate a local worker and lose any completion callback | [`spec:obligations.obligation-module`](specs/obligations/obligation-module.sdp.md) | [`.duplicate-worker-lost-callback`](specs/obligations/obligation-module.duplicate-worker-lost-callback.sdp.md) | native | | L3-2 | Kill a dispatch before work, or fail the scheduled wrapper | [`spec:obligations.obligation-module`](specs/obligations/obligation-module.sdp.md) | [`.dispatch-killed-rearmed`](specs/obligations/obligation-module.dispatch-killed-rearmed.sdp.md), [`.wrapper-failed-rearmed`](specs/obligations/obligation-module.wrapper-failed-rearmed.sdp.md) | native | | L3-3 | Legitimate backlog builds up | [`spec:obligations.obligation-module`](specs/obligations/obligation-module.sdp.md) | [`.backlog-left-alone`](specs/obligations/obligation-module.backlog-left-alone.sdp.md) | native | diff --git a/design/specs/application/first-experiment.native-contention.sdp.md b/design/specs/application/first-experiment.native-contention.sdp.md new file mode 100644 index 0000000..cb9d5c2 --- /dev/null +++ b/design/specs/application/first-experiment.native-contention.sdp.md @@ -0,0 +1,36 @@ +--- +id: spec:application.first-experiment.native-contention +kind: example +altitude: story +readiness: idea +relations: + refines: spec:application.first-experiment + verifies: spec:application.first-experiment +--- +# Concurrent commands on the production composition + +Sc L2-9 · native tier · production composition · Traces: First experiment, Sc L1-11, Sc L2-9, E-47. + +The concurrency half of native acceptance observes the production composition on the pinned local backend. The separate production-configuration example carries the code-path half. Neither claims parity with a hosted deployment. + +## Intent + +- outcome: On the local native backend, concurrent orders share stock for one; one is applied, the others reach short-stock rejections at two and eight callers, and at thirty-two callers engine failures are recorded separately (Sc L2-9, Sc L1-11, First experiment) + +```gwt +Given the production composition on a native backend +And the contention matrix has sizes {sizes: "1, 10, 100"} and callers {callers: "2, 8, 32"} +And the backend runs with {configuration: "production configuration"} +When {run: "the PlaceOrder use case"} runs +Then each successful command makes {commits: 1} top-level commit +And the core path runs {projectionJobs: 0} projection jobs +And the contention matrix records one applied outcome per cell and {smallCellEngineFailures: 0} engine failures at two and eight callers +``` + +## Verification — executable + +- Runs in the native backend tier on the production composition, with an ordinary authenticated client, four grants, one active generation and a disposable local backend per cell; the client sends commands concurrently without its mutation queue. +- The only stock item shared by the commands has quantity one and is first in every order; all remaining lines name items belonging to one command, each received once with quantity two. Each record states this layout. +- At each size, two and eight callers leave one order, one receipt and one summary, and all other commands are rejected for short stock; thirty-two callers leave the same applied effect and report short-stock rejections and engine failures separately. +- Each command is counted by its backend request ID with all completion records retained; a record with `willRetry` true and `occInfo` is an execution refused at commit, even when its error field is null. Terminal records and caller answers are counted separately, and no caller retries during the cell. +- The verifier asserts documents and bytes from each completion record, one applied order's retained effects, no scheduled functions and the immediate essential summary, while the number of engine reruns and the tables they name remain observations. diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index ee725a0..ef1fd98 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -98,20 +98,29 @@ The six cost targets are the constraint Specs this workflow is constrained by. T The experiment is an application, a test suite and a report. The application is the production composition: two context components, the parent with its pipeline, use cases, read models, registry and gate. The suite is the acceptance scenarios of Layers 0 to 2, run against a disposable local backend as the acceptance contract prescribes. The report is a set of measurement records, one per run, compared against the six constraint targets. The production composition is the Convex app whose project directory is `example/`: its functions are in `example/convex/`, its two contexts are the components `example/convex/orders/` and `example/convex/inventory/`, and its deciders are in `example/domain/`. -- typeMeasurement: `[extension] interface Measurement { commit: string; backendVersion: string; convexVersion: string; configuration: "production" | "adjusted"; adjustments?: string; identitySource: "production-issuer" | "fixture-issuer"; dataset: string; command: "PlaceOrder" | "CancelOrder"; lines: number; contention: number; path: "healthy" | "retry"; topLevelCommits: number; functionCalls: number; componentCalls: number; documentsRead: number; documentsWritten: number; bytesRead: number; bytesWritten: number; rowsLeftBehind: number; occRetries: number | "not exposed"; occExhaustions: number; latencyMs: number }` where `contention` is the number of concurrent callers and `occExhaustions` counts commands the engine failed with its documents-changed error (E-47, E-13, First experiment) -- captureCommits: top-level commits are the successful top-level mutation executions in the backend's function log for the run (E-47, First experiment) -- captureCalls: function calls are the top-level executions the backend's function log shows; the log holds no record for a component call inside a mutation, so the calls a use case makes per context are counted by a pure test that runs the use case's executor against a ctx whose `runMutation` counts by function reference and answers a canned operation outcome (E-47, F4, E-15) -- captureDocuments: documents and bytes read and written come from the `usageStats` of the command's top-level completion record, `databaseReadDocuments`, `databaseWriteDocuments`, `databaseReadBytes` and `databaseWriteBytes`, which count what the component calls inside the mutation read and wrote; a count the backend does not expose is recorded as `not exposed` (E-47, First experiment, E-15) -- captureRowsLeftBehind: rows left behind are the documents a run created that are not state, events, receipts or read-model rows: markers, registry rows, gate and audit records, counted by table (E-47, First experiment, D19) -- captureRetries: optimistic-concurrency retries are not observable from inside a mutation; the contention run reports them from the backend's log where exposed, else the retry path is measured as the difference between the contention run and the healthy run (E-47, F1) +- typeMeasurement: [extension] `type Measurement = { recordType: "Measurement"; commit: string; pins: MeasurementPins; configuration: "production" | "adjusted"; adjustments?: string; identitySource: "fixture-issuer" | "production-issuer"; installedLayers: string[]; dataset: string; command: "PlaceOrder" | "CancelOrder"; lines: number; contention: number; path: string; submittedCommands: number; executions: number; finalOutcomes: number; outcomeCounts: Record; topLevelCommits: number; functionCalls: number; componentCalls: number | "not exposed"; rolledBackWrites: number | "not exposed"; writeSetFromLog: "not exposed"; usageStats: MeasurementUsage; finalUsageStats: MeasurementUsage; rerunUsageStats: MeasurementUsage; engineReruns: number; engineFailedCommands: number; occTables: string[]; rowsBefore: Record; rowsAfter: Record; rowsLeftBehind: Record; requests: MeasurementRequest[]; callerAnswers: unknown[]; requestToInput: string; writeCapBytesPerSecond: number; timingReading: "taken under load" | "quiet machine"; latencyDistribution: unknown; throughput: unknown }`; the native helper names this record `PlaceOrderMeasurement`, apart from the harness name/value envelope named `Measurement` (E-47, E-13, First experiment) +- typeMeasurementPins: [extension] `type MeasurementPins = { tier: "native backend"; environment: "local backend"; backend: { release: string; sha256: string; source: string }; convex: string; convexHelpers: string; convexTest: string; node: string; operatingSystem: string }` records backend, dependencies, Node and operating system in every cell; no local number claims a hosted result (E-47, E-15, Sc L2-9) +- typeMeasurementUsage: [extension] `type MeasurementUsage = { databaseReadDocuments: number; databaseWriteDocuments: number; databaseReadBytes: number; databaseWriteBytes: number; databaseWriteIndexRows: number }` carries only counters the native backend recorded (E-47, E-15) +- typeMeasurementRequest: [extension] `type MeasurementRequest = { requestId: string; executions: number; finalOutcomes: number; finalError: string | null; engineReruns: number; occTables: string[]; usageStats: MeasurementUsage; finalUsageStats: MeasurementUsage; rerunUsageStats: MeasurementUsage; latencyMs: number; completionRecords: unknown[] }` keeps every completion in its backend request; concurrent HTTP answers are retained separately because the response exposes no backend request ID, so their order never supplies a guessed correspondence (E-47, E-15) +- captureTiming: [extension] latency is the backend log's top-level `executionTime`, converted to milliseconds, over thirty accepted commands per size after five discarded; the median averages the middle pair and the 95th percentile uses nearest rank; throughput is applied ten-line orders divided by elapsed wall time over at least ten seconds with eight callers on mutually disjoint stock, with the local write cap of 4194304 bytes per second recorded beside it; poll the function log throughout the throughput cell and fail if any poll may have lost entries or the count of terminal records differs from submitted commands (E-47, F13) +- captureCommits: top-level commits are terminal top-level mutation completion records with `willRetry` false and `error` null; a completion with `willRetry` true is no commit even when `error` is null, and an authorized duplicate may commit a read-only transaction with no new business effect (E-47, First experiment) +- captureCalls: function calls count the command's top-level completion records, including executions the engine reran; component calls and their individual time are `not exposed` by the native backend log, and a separate pure test of the use case executor counts calls by context with canned outcomes at each size (E-47, F4, E-15) +- captureDocuments: documents and bytes come only from each top-level completion record's `usageStats`; the record retains every execution and sums its recorded counters by backend request ID and by cell, separately for terminal executions and executions refused at commit, without deriving rolled-back writes or a write set from stored rows; a field the backend does not expose is `not exposed` (E-47, First experiment, E-15) +- captureRowsLeftBehind: rows left behind are the documents a run creates that are not state, events, receipts or read-model rows; record their before, after and created counts by table, separately from setup, including markers, registry, gate and audit rows, and say `not deployed` for a table absent from the deployed schema (E-47, First experiment, D19) +- captureRetries: one completion with `willRetry` true and an `occInfo` is one execution refused at commit; its rerun has another completion with the same backend request ID; record these executions, the tables named by `occInfo`, each command's terminal record and final outcomes separately, and count terminal optimistic-concurrency failures as commands the engine failed; the variable number of reruns is an observation, never the difference between two cells (E-47, F1) - datasetSizes: 1 line, 10 lines and 100 lines as the provisional maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) -- contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with the stock set so that the first is applied and the rest are rejected for short stock, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry (First experiment, Sc L1-11, F1, E-5, E-47) +- contentionShape: at each size, send N `PlaceOrder` commands concurrently at N of 2, 8 and 32, sharing only one stock item with quantity one; state the placement of that item and the layout of the other lines in the dataset; at N of 2 and 8 require one applied outcome, N minus one short-stock rejections and zero commands the engine failed, while at N of 32 report those counts and engine failures without treating an engine failure as an experiment failure; no caller retries during the cell, and the caller may repeat a failed command under its same request key (First experiment, Sc L1-11, F1, E-5, E-47) - limitOrderLines: the maximum number of lines per `PlaceOrder` is a product decision, open under OQ3; the experiment measures at 100 and reports documents read and written per line, so the owner can set it under the 1 second timeout and the F13 ceilings (OQ3, F13) - limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate read, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer (E-47, F13) +- observedMeasurementPins: [extension] the measurement records name the native backend tier, the local backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124, `convex-test` 0.0.60, Node v24.19.0 and Linux 7.2.7-200.fc44.x86_64 on x64; these observations concern that local environment only (E-47, E-15, Sc L2-9) +- observedMeasurementBytes: [extension] at the native backend tier with `observedMeasurementPins`, fixed seven-character stock and order IDs, four grants and one active generation, orders of 1, 10 and 100 distinct stock items received once read 2943, 14454 and 129564 bytes and write 4271, 18410 and 159800 bytes; the corresponding write-index rows are 25, 97 and 817; a ten-line authorized duplicate or changed-input conflict reads 5 documents and 3441 bytes and writes zero, while short stock or a missing journal tail on the last line reads 32 documents and 13106 bytes and writes zero (E-47, F13) +- observedContentionCosts: [extension] at the native backend tier with `observedMeasurementPins`, each of the nine cells at 1, 10 and 100 lines and 2, 8 and 32 callers reached one applied outcome and all remaining short-stock rejections with zero engine failures; an execution refused at commit read the healthy path's documents and bytes and wrote zero, and its final short-stock rejection, with the shared item first, read 5 documents and 1595 bytes and wrote zero; `occInfo` named Inventory's events or streams across the cells, and the rerun count remains an observation per record (E-47, F1) +- observedUnrelatedStock: [extension] at the native backend tier with `observedMeasurementPins`, eight simultaneous ten-line commands on disjoint stock in one tenant and in eight unrelated tenants each recorded 8 executions, 8 applied outcomes and zero engine reruns; each command read 35 documents and 14454 bytes and wrote 24 documents and 18410 bytes, and the shared generation row was unchanged (E-47, F1) +- observedStockFoldGap: [extension] at the native backend tier with `observedMeasurementPins`, after `ReceiveStock` recorded quantity two, an admin change of current on-hand stock to three with the same stream version and journal tail let a one-line order allocate three, reading 8 documents and 2943 bytes and writing 6 documents and 4271 bytes; matching the tail version does not establish equality of current state with the complete event fold (E-47, D3) - streamBudgets: the stock item stream declares `budgetBytes` of 16 KiB and the order stream keeps the default 256 KiB, as `spec:application.orders-inventory-example` pins, so the allocation call's byte bound, 8 MiB over the planned streams' budgets, admits 512 stock streams and the adapter's count bound of `min(maxStreams, 256)` admits the 100 of the maximum order with room, while the single order stream costs at most 256 KiB; at the default budget on both stream types the byte bound would admit only 32 stock streams and the maximum run would be rejected `operationTooLarge`, which is why the budget is pinned here and the run at the maximum measures it (E-2, E-22, E-46, F13, Sc L2-3) - productDecisions: latency and throughput targets are set by the owner before the benchmark and recorded with the run; the experiment reports latency but does not set a target (First experiment) -- deliverables: the application in this repository, beside the design, which settles OQ4, the evidence records of every scenario run, and the measurement report with the six targets marked held or not held (First experiment, OQ4, Acceptance scenarios) +- deliverables: the application, native evidence records and measurement report with the six cost targets marked held or not held against measured fields; one explicit timing command records the latency distribution after discarded warmups and throughput with its write cap, prints each fixed speed target as held or not held, and labels readings taken under load for replacement by a quiet-machine reading; speed misses are report values, not failing assertions (First experiment, OQ4, Acceptance scenarios, E-47) - authProvider: [extension] the production composition's `auth.config.ts` declares one `customJwt` provider whose `issuer`, `applicationID` and `jwks` are environment variables of the deployment, as `spec:platform.native-harness` pins for a composition that runs natively (E-13, D11) - acceptanceRows: [extension] the rows the experiment's acceptance check requires are Sc L0-1, Sc L0-2, Sc L1-1, Sc L1-2, Sc L1-3, Sc L1-4, Sc L1-5, Sc L1-6, Sc L1-7, Sc L1-8, Sc L1-9, Sc L1-10, Sc L1-11, Sc L1-12, Sc L2-1, Sc L2-2, Sc L2-3, Sc L2-4, Sc L2-5, Sc L2-6, Sc L2-7, Sc L2-8 and Sc L2-9, which are the rows the doc's table gives Layer 0, 1 or 2 (E-17, First experiment, Acceptance scenarios) @@ -120,12 +129,14 @@ The experiment is an application, a test suite and a report. The application is ```gwt-vocabulary Given the production composition on a native backend And an order of {size:"1 line"|"10 lines"|"the maximum"} with stock contention {contention:"absent"|"present"} +And the contention matrix has sizes {sizes:string} and callers {callers:string} And the backend runs with {configuration:"production configuration"|"test configuration"} When {run:"the PlaceOrder use case"|"the end-to-end path"} runs Then each successful command makes {commits:number} top-level commit And the core path runs {projectionJobs:number} projection jobs And the use case makes {callsPerContext:number} call per context And the budgets {budgets:"hold"|"are exceeded"} +And the contention matrix records one applied outcome per cell and {smallCellEngineFailures:number} engine failures at two and eight callers And authority, schemas and code path {parity:"match a release"|"differ from a release"} ``` diff --git a/package.json b/package.json index a0e7ea9..c2e858c 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,8 @@ "test:all": "npm run sdp:build && vitest run", "acceptance": "node scripts/acceptance.mjs", "codegen": "node scripts/codegen.mjs", - "dev": "node scripts/dev.mjs" + "dev": "node scripts/dev.mjs", + "measure:place-order": "node scripts/measure-place-order.mjs" }, "devDependencies": { "@convex-dev/eslint-plugin": "5.0.0", diff --git a/scripts/measure-place-order.mjs b/scripts/measure-place-order.mjs new file mode 100644 index 0000000..43f020d --- /dev/null +++ b/scripts/measure-place-order.mjs @@ -0,0 +1,32 @@ +import { spawnSync } from "node:child_process"; +import { homedir } from "node:os"; +import { join } from "node:path"; +const environment = { ...process.env }; +for (const key of [ + "CONVEX_DEPLOY_KEY", + "CONVEX_DEPLOYMENT_TOKEN", + "CONVEX_DEPLOYMENT", +]) + delete environment[key]; +environment.PLACE_ORDER_TIMINGS = "1"; +environment.PLACE_ORDER_TIMING_LOAD = process.argv.includes("--quiet") + ? "quiet" + : "under load"; +environment.CONVEX_BACKEND_BINARY = join( + homedir(), + "dev-libar/libar-platform/.cache/precompiled-2026-09-28-5c7cb5b/convex-local-backend-x86_64-unknown-linux-gnu.zip/convex-local-backend", +); +const child = spawnSync( + process.execPath, + [ + "node_modules/vitest/vitest.mjs", + "run", + "--project", + "native", + "--maxWorkers=1", + "tests/native/place-order-timing.test.ts", + ], + { stdio: "inherit", env: environment }, +); +if (child.error) throw child.error; +process.exitCode = child.status ?? 1; diff --git a/tests/native/place-order-contention.test.ts b/tests/native/place-order-contention.test.ts new file mode 100644 index 0000000..35831c2 --- /dev/null +++ b/tests/native/place-order-contention.test.ts @@ -0,0 +1,65 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { nativeContentionContract as contract } from "../../generated/contracts/application.first-experiment.native-contention.contract.js"; +import { contentionCell } from "./place-order-contention.js"; +import type { PlaceOrderMeasurement } from "./place-order-measurement.js"; +const anchor = specTest({ + id: testAnchorId("test:application.first-experiment.native-contention"), + verifies: ref("spec:application.first-experiment.native-contention"), +}); +void anchor; +bindExample( + contract, + (): { cells: PlaceOrderMeasurement[] } => ({ cells: [] }), + { + "the production composition on a native backend": () => {}, + "the contention matrix has sizes {sizes} and callers {callers}": ( + _world, + { sizes, callers }, + ) => { + expect(sizes).toBe("1, 10, 100"); + expect(callers).toBe("2, 8, 32"); + }, + "the backend runs with {configuration}": (_world, { configuration }) => { + expect(configuration).toBe("production configuration"); + }, + "{run} runs": async (world, { run }) => { + expect(run).toBe("the PlaceOrder use case"); + for (const lines of [1, 10, 100] as const) + for (const callers of [2, 8, 32] as const) + world.cells.push(await contentionCell(lines, callers)); + }, + "each successful command makes {commits} top-level commit": ( + world, + { commits }, + ) => { + expect(world.cells).toHaveLength(9); + for (const cell of world.cells) + expect(cell.topLevelCommits).toBe(commits); + }, + "the core path runs {projectionJobs} projection jobs": ( + world, + { projectionJobs }, + ) => { + for (const cell of world.cells) + expect((cell.otherExecutions as unknown[]).length).toBe(projectionJobs); + }, + "the contention matrix records one applied outcome per cell and {smallCellEngineFailures} engine failures at two and eight callers": + (world, { smallCellEngineFailures }) => { + for (const cell of world.cells) { + expect(cell.outcomeCounts.applied).toBe(1); + if (cell.contention < 32) + expect(cell.engineFailedCommands).toBe(smallCellEngineFailures); + } + expect( + world.cells.reduce((sum, cell) => sum + cell.engineReruns, 0), + "the matrix must observe at least one refused commit", + ).toBeGreaterThan(0); + }, + }, +); diff --git a/tests/native/place-order-contention.ts b/tests/native/place-order-contention.ts new file mode 100644 index 0000000..920bba7 --- /dev/null +++ b/tests/native/place-order-contention.ts @@ -0,0 +1,100 @@ +import { expect } from "vitest"; +import { required } from "../../harness/native.js"; +import { + healthyUsage, + expectUsage, + measured, + order, + seed, + setup, +} from "./place-order-measurement.js"; +export async function contentionCell(lines: 1 | 10 | 100, callers: 2 | 8 | 32) { + const world = await setup(); + const shared = order(1); + await seed(world, shared, 1); + const args = Array.from({ length: callers }, (_, index) => { + const arg = order(lines, index); + required(arg.input.lines[0], "first line").stockItemId = "sku-000"; + return arg; + }); + for (const arg of args) + if (lines > 1) + await seed(world, { + ...arg, + input: { ...arg.input, lines: arg.input.lines.slice(1) }, + }); + const result = await measured( + world, + args, + `${callers} simultaneous commands, each ${lines} lines; only sku-000 is shared and has quantity one; it is first in each list; every other stock item belongs to one command and was received once with quantity two; four grants, one active generation; seven-character stock and order IDs.`, + "contention", + callers, + ); + expect(result.value.outcomeCounts.applied).toBe(1); + expect(result.value.outcomeCounts.duplicates).toBe(0); + expect(result.value.outcomeCounts.businessFailure).toBe(0); + expect(result.value.topLevelCommits).toBe(1); + const stockItems = 1 + (lines - 1) * callers; + expect(result.value.rowsBefore).toEqual({ + grants: 4, + generations: 1, + receipts: 0, + summaries: 0, + orderStreams: 0, + orderEvents: 0, + stockStreams: stockItems, + stockEvents: stockItems, + }); + expect(result.value.rowsAfter).toEqual({ + grants: 4, + generations: 1, + receipts: 1, + summaries: 1, + orderStreams: 1, + orderEvents: 1, + stockStreams: stockItems, + stockEvents: stockItems + lines, + }); + expect( + result.value.outcomeCounts.rejected + result.value.engineFailedCommands, + ).toBe(callers - 1); + if (callers !== 32) { + expect(result.value.outcomeCounts.rejected).toBe(callers - 1); + expect(result.value.engineFailedCommands).toBe(0); + } + expect( + result.answers + .filter((answer) => answer.kind === "rejection") + .map((answer) => answer.code), + ).toEqual( + Array(result.value.outcomeCounts.rejected).fill("insufficientStock"), + ); + for (const record of result.records) { + if (record.willRetry) { + const [reads, , bytes] = healthyUsage[lines]; + expectUsage(record, [reads, 0, bytes, 0]); + } else if (record.error === null) expectUsage(record, healthyUsage[lines]); + else if (!record.willRetry && record.occInfo === null) + expectUsage(record, [5, 0, 1595, 0]); + } + expect(result.after.receipts.length - result.before.receipts.length).toBe(1); + expect( + result.after.orderStreams.length - result.before.orderStreams.length, + ).toBe(1); + expect( + result.after.orderEvents.length - result.before.orderEvents.length, + ).toBe(1); + expect( + result.after.stockEvents.length - result.before.stockEvents.length, + ).toBe(lines); + expect(result.after.summaries.length - result.before.summaries.length).toBe( + 1, + ); + const stock = required( + result.after.stockStreams.find((row) => row.streamId === "sku-000"), + "shared stock", + ); + expect(stock.state).toEqual({ onHand: 1, allocated: 1 }); + await required(world.backend, "backend").dispose(); + return result.value; +} diff --git a/tests/native/place-order-measurement.test.ts b/tests/native/place-order-measurement.test.ts new file mode 100644 index 0000000..a368060 --- /dev/null +++ b/tests/native/place-order-measurement.test.ts @@ -0,0 +1,252 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { expect, test } from "vitest"; +import { measure, required } from "../../harness/native.js"; +import { + place, + callsPerContextAre, + type ExperimentWorld, +} from "./first-experiment-steps.js"; +import { stored } from "./place-order-attribution.js"; +import { + expectUsage, + json, + grantTenant, + healthyUsage, + measured, + order, + seed, + setup, +} from "./place-order-measurement.js"; +const anchor = specTest({ + id: testAnchorId("test:application.first-experiment.measurement"), + verifies: ref("spec:application.first-experiment"), +}); +void anchor; + +test.each([1, 10, 100] as const)( + "native: PlaceOrder of %i lines has the recorded document and byte cost", + async (lines) => { + const world = await setup(); + const args = order(lines); + await seed(world, args); + const result = await measured( + world, + [args], + "Four grants, one active generation; each line names a distinct stock item received once with quantity two; fixed seven-character stock and order IDs.", + "healthy", + ); + expect(result.answers[0]).toMatchObject({ + kind: "applied", + replayed: false, + }); + expect(result.records).toHaveLength(1); + expectUsage(required(result.records[0], "completion"), healthyUsage[lines]); + expect(result.value.rowsAfter).toEqual({ + grants: 4, + generations: 1, + receipts: 1, + summaries: 1, + orderStreams: 1, + orderEvents: 1, + stockStreams: lines, + stockEvents: 2 * lines, + }); + expect( + result.after.orderEvents.length - result.before.orderEvents.length, + ).toBe(1); + expect( + result.after.stockEvents.length - result.before.stockEvents.length, + ).toBe(lines); + expect(result.after.summaries.length - result.before.summaries.length).toBe( + 1, + ); + // Reuse the executor's pure-tier call counter with the actual input and returned operation. + const response = result.answers[0]?.value as NonNullable< + ExperimentWorld["placed"] + >["response"]; + world.placed = { + orderId: args.input.orderId, + lines: args.input.lines, + response, + own: required(result.records[0], "completion"), + request: result.records, + window: result.records, + summaryKeys: [args.input.orderId], + }; + await callsPerContextAre(world, 1); + measure("callsPerContext", { + tier: "pure test", + lines, + orders: 1, + inventory: 1, + }); + }, +); + +test.each([ + "authorized duplicate", + "changed input conflict", + "business rejection", + "second-context failure", +] as const)("native: PlaceOrder cost for %s", async (path) => { + const world = await setup(); + const backend = required(world.backend, "backend"); + const args = order(10); + await seed(world, args); + let priorCommand = null; + if (path === "authorized duplicate" || path === "changed input conflict") { + const original = await place(world, args.input.orderId, args.input.lines); + expect(original.response).toMatchObject({ + kind: "applied", + replayed: false, + }); + expectUsage(original.own, healthyUsage[10]); + priorCommand = json({ + outcome: original.response, + completionRecords: original.request, + }); + if (path === "changed input conflict") + required(args.input.lines[9], "last line").unitPrice++; + } + if (path === "business rejection") + required(args.input.lines[9], "last line").quantity = 3; + if (path === "second-context failure") { + const rows = await stored(backend); + const event = required( + rows.stockEvents.find((row) => row.streamId === "sku-009"), + "tail", + ); + await backend.admin.writeTable( + "events", + { delete: String(event._id) }, + { component: "inventory" }, + ); + } + const result = await measured( + world, + [args], + `Ten distinct stock items, four grants and one active generation; ${path}; the short-stock or missing-tail item is last, after nine Inventory allocations and the Orders call.`, + path, + 0, + { priorCommand }, + ); + expect(result.records).toHaveLength(1); + const expected = + path === "authorized duplicate" || path === "changed input conflict" + ? ([5, 0, 3441, 0] as const) + : ([32, 0, 13106, 0] as const); + expectUsage(required(result.records[0], "completion"), expected); + expect(result.after).toEqual(result.before); + if (path === "authorized duplicate") + expect(result.answers[0]).toMatchObject({ + kind: "applied", + replayed: true, + }); + if (path === "changed input conflict") + expect(result.answers[0]).toMatchObject({ + kind: "rejection", + code: "idempotencyConflict", + }); + if (path === "business rejection") + expect(result.answers[0]).toMatchObject({ + kind: "rejection", + code: "insufficientStock", + }); + if (path === "second-context failure") { + expect(result.answers[0]?.kind).toBe("technicalFailure"); + expect(JSON.stringify(result.answers[0]?.value)).toContain( + "journal holds no event", + ); + } +}); + +test.each([false, true])( + "native: unrelated stock commands with separate tenants %s", + async (separateTenants) => { + const world = await setup(); + const args = Array.from({ length: 8 }, (_, i) => + order(10, i, separateTenants ? `t-${i + 1}` : "t-1"), + ); + for (const arg of args) { + if (separateTenants && arg.tenantId !== "t-1") + await grantTenant(world, arg.tenantId); + await seed(world, arg); + } + const result = await measured( + world, + args, + `Eight simultaneous commands of ten lines; no shared stock item; ${separateTenants ? "eight unrelated tenants, each with four grants" : "one tenant with four grants"}; one active generation, seven-character stock and order IDs.`, + "unrelated stock", + 8, + ); + expect(result.answers.map((answer) => answer.kind)).toEqual( + Array(8).fill("applied"), + ); + expect(result.value.engineFailedCommands).toBe(0); + expect(result.value.topLevelCommits).toBe(8); + expect(result.value.rowsAfter).toEqual({ + grants: separateTenants ? 32 : 4, + generations: 1, + receipts: 8, + summaries: 8, + orderStreams: 8, + orderEvents: 8, + stockStreams: 80, + stockEvents: 160, + }); + for (const record of result.records.filter((record) => !record.willRetry)) + expectUsage(record, healthyUsage[10]); + expect( + result.after.orderEvents.length - result.before.orderEvents.length, + ).toBe(8); + expect( + result.after.stockEvents.length - result.before.stockEvents.length, + ).toBe(80); + expect(result.after.receipts.length - result.before.receipts.length).toBe( + 8, + ); + }, +); + +test("native: a matching journal tail does not prove that current stock equals its event fold", async () => { + const world = await setup(); + const backend = required(world.backend, "backend"); + const args = order(1); + await seed(world, args); + const before = await stored(backend); + const stock = required(before.stockStreams[0], "stock"); + await backend.admin.writeTable( + "streams", + { + patch: String(stock._id), + fields: { state: { onHand: 3, allocated: 0 } }, + }, + { component: "inventory" }, + ); + required(args.input.lines[0], "line").quantity = 3; + const result = await measured( + world, + [args], + "One stock item received with quantity two; admin access changes only current onHand to three, keeping stream version one and its journal tail; PlaceOrder asks for three. This corrupt state is not made by a command.", + "current state differs from fold", + ); + expect(result.answers[0]).toMatchObject({ kind: "applied", replayed: false }); + expectUsage(required(result.records[0], "completion"), healthyUsage[1]); + expect(result.after.stockStreams[0]?.state).toEqual({ + onHand: 3, + allocated: 3, + }); + expect( + result.after.stockEvents.map((row) => ({ + type: row.eventType, + payload: row.payload, + })), + ).toEqual([ + { type: "StockReceived", payload: { quantity: 2 } }, + { type: "StockAllocated", payload: { quantity: 3, orderId: "order-1" } }, + ]); +}); diff --git a/tests/native/place-order-measurement.ts b/tests/native/place-order-measurement.ts new file mode 100644 index 0000000..4c366f6 --- /dev/null +++ b/tests/native/place-order-measurement.ts @@ -0,0 +1,457 @@ +import { execFileSync } from "node:child_process"; +import { arch, platform, release } from "node:os"; +import { readFileSync } from "node:fs"; +import { getFunctionName, type FunctionReturnType } from "convex/server"; +import { ConvexError } from "convex/values"; +import { expect } from "vitest"; +import { api, internal } from "../../example/convex/_generated/api.js"; +import type { CompletionRecord, LogMark } from "../../harness/admin.js"; +import type { Backend } from "../../harness/backend.js"; +import type { JsonValue } from "../../harness/evidence.js"; +import { measure, required } from "../../harness/native.js"; +import { localWriteRateBytesPerSecond } from "../../harness/local-write-rate.js"; +import { + application, + configurationIs, + grantedPermissions, + orderLines, + placeOrderIdentifier, + subject, + tenantId, + usageKeys, + usageOf, + type ExperimentWorld, + type Usage, +} from "./first-experiment-steps.js"; +import { + rowCounts, + stored, + type OrderArgs, + type Stored, +} from "./place-order-attribution.js"; + +export type ExecutionRecord = CompletionRecord & { + willRetry: boolean; + occInfo: { tableName: string } | null; +}; +export type Answer = { + kind: string; + code: string | null; + replayed: boolean; + value: JsonValue; +}; +export const json = (value: unknown): JsonValue => + JSON.parse(JSON.stringify(value)) as JsonValue; +export const pad = (n: number) => String(n).padStart(3, "0"); +export const writeCap = localWriteRateBytesPerSecond; +export async function setup(): Promise { + const world: ExperimentWorld = {}; + await application(world); + await configurationIs(world, "production configuration"); + return world; +} +export async function seed( + world: ExperimentWorld, + args: OrderArgs, + quantity = 2, +) { + const client = required(world.client, "client"); + for (let offset = 0; offset < args.input.lines.length; offset += 100) + await client.mutation(api.receiving.receiveStock, { + tenantId: args.tenantId, + input: { + items: args.input.lines + .slice(offset, offset + 100) + .map((line) => ({ stockItemId: line.stockItemId, quantity })), + }, + }); +} +export async function grantTenant(world: ExperimentWorld, tenant: string) { + const backend = required(world.backend, "backend"); + for (const permission of grantedPermissions) + await backend.admin.run(getFunctionName(internal.grants.grant), { + tenantId: tenant, + principalKind: "human", + principalId: `${backend.issuer.issuer}|${subject}`, + permission, + grantedBy: "native-test", + }); +} +export function order( + lines: number, + index?: number, + tenant = tenantId, +): OrderArgs { + const orderId = index === undefined ? "order-1" : `ord-${pad(index)}`; + return { + tenantId: tenant, + requestKey: `k-${orderId}`, + input: { + orderId, + lines: orderLines(lines, index === undefined ? "sku" : pad(index)), + }, + }; +} +export async function send( + world: ExperimentWorld, + args: OrderArgs, +): Promise { + try { + const value = await required(world.client, "client").mutation( + api.ordering.placeOrder, + args, + { skipQueue: true }, + ); + return { + kind: value.kind, + code: null, + replayed: value.replayed, + value: json(value), + }; + } catch (error) { + const value = + error instanceof ConvexError + ? json(error.data) + : { message: String(error) }; + const data = value as { kind?: string; code?: string }; + return { + kind: data.kind ?? "technicalFailure", + code: data.code ?? null, + replayed: false, + value, + }; + } +} +export const isOwn = (record: CompletionRecord) => + record.identifier === placeOrderIdentifier && record.componentPath === null; +export const finals = (records: readonly CompletionRecord[]) => + records + .filter(isOwn) + .filter((record) => (record as ExecutionRecord).willRetry === false); +export function collect( + backend: Backend, + mark: LogMark, + count: () => number, + done: () => boolean = () => true, + timeout = 30000, +) { + return backend.admin.completionsSince( + mark, + (records) => done() && finals(records).length >= count(), + timeout, + ); +} +const sumUsage = (records: readonly CompletionRecord[]): Usage => + Object.fromEntries( + usageKeys.map((key) => [ + key, + records.reduce((sum, record) => sum + usageOf(record)[key], 0), + ]), + ) as Usage; +const packageVersion = (name: string) => + ( + JSON.parse( + readFileSync( + new URL(`../../node_modules/${name}/package.json`, import.meta.url), + "utf8", + ), + ) as { version: string } + ).version; +export function pins(backend: Backend) { + return { + tier: "native backend", + environment: "local backend", + backend: backend.facts().executable, + convex: packageVersion("convex"), + convexHelpers: packageVersion("convex-helpers"), + convexTest: packageVersion("convex-test"), + node: process.version, + operatingSystem: `${platform()} ${release()} ${arch()}`, + }; +} +export async function schedules(backend: Backend) { + const entries = await Promise.all( + [undefined, "orders", "inventory"].map( + async (component) => + [ + component ?? "parent", + ( + await backend.admin.readTable( + "_scheduled_functions", + component === undefined ? {} : { component }, + ) + ).length, + ] as const, + ), + ); + return Object.fromEntries(entries); +} +export function leftBehind(before: Stored, after: Stored) { + return { + generations: { + before: before.generations.length, + after: after.generations.length, + created: after.generations.filter( + (row) => !before.generations.some((prior) => prior._id === row._id), + ).length, + }, + markers: "not deployed", + gate: "not deployed", + audit: "not deployed", + }; +} +// A complete experiment record, distinct from the harness's name/value Measurement envelope. +export function recordMeasurement( + world: ExperimentWorld, + options: { + dataset: string; + lines: number; + contention: number; + path: string; + answers: Answer[]; + records: CompletionRecord[]; + before: Stored; + after: Stored; + extra?: Record; + }, +) { + const backend = required(world.backend, "backend"); + const own = options.records.filter(isOwn) as ExecutionRecord[]; + const completed = finals(own) as ExecutionRecord[]; + const requests = [...new Set(own.map((record) => record.requestId))].map( + (requestId) => { + const executions = own.filter((record) => record.requestId === requestId); + const final = required( + executions.find((record) => record.willRetry === false), + "final completion", + ); + const reruns = executions.filter((record) => record.willRetry); + return { + requestId, + executions: executions.length, + finalOutcomes: 1, + finalError: final.error, + engineReruns: reruns.length, + occTables: [ + ...new Set( + executions.flatMap((record) => + record.occInfo === null ? [] : [record.occInfo.tableName], + ), + ), + ], + usageStats: sumUsage(executions), + finalUsageStats: usageOf(final), + rerunUsageStats: sumUsage(reruns), + latencyMs: final.executionTime * 1000, + completionRecords: executions, + }; + }, + ); + const outcomeCounts = { + applied: options.answers.filter( + (answer) => answer.kind === "applied" && !answer.replayed, + ).length, + duplicates: options.answers.filter((answer) => answer.replayed).length, + rejected: options.answers.filter((answer) => answer.kind === "rejection") + .length, + businessFailure: options.answers.filter( + (answer) => answer.kind === "businessFailure", + ).length, + technicalFailure: options.answers.filter( + (answer) => answer.kind === "technicalFailure", + ).length, + transientRefusal: options.answers.filter( + (answer) => answer.kind === "transient", + ).length, + }; + const value = { + recordType: "Measurement", + commit: execFileSync("git", ["rev-parse", "HEAD"], { + encoding: "utf8", + }).trim(), + pins: pins(backend), + configuration: "production", + identitySource: "fixture-issuer", + installedLayers: backend.facts().installedLayers, + dataset: options.dataset, + command: "PlaceOrder", + lines: options.lines, + contention: options.contention, + path: options.path, + otherExecutions: options.records.filter( + (record) => + !isOwn(record) && record.identifier !== "readModels:listOrderSummaries", + ), + submittedCommands: options.answers.length, + executions: own.length, + finalOutcomes: completed.length, + outcomeCounts, + topLevelCommits: completed.filter((record) => record.error === null).length, + functionCalls: own.length, + componentCalls: "not exposed", + rolledBackWrites: "not exposed", + writeSetFromLog: "not exposed", + usageStats: sumUsage(own), + finalUsageStats: sumUsage(completed), + rerunUsageStats: sumUsage(own.filter((record) => record.willRetry)), + engineReruns: own.filter((record) => record.willRetry).length, + engineFailedCommands: completed.filter((record) => record.occInfo !== null) + .length, + occTables: [ + ...new Set( + own.flatMap((record) => + record.occInfo === null ? [] : [record.occInfo.tableName], + ), + ), + ], + rowsBefore: rowCounts(options.before), + rowsAfter: rowCounts(options.after), + rowsLeftBehind: leftBehind(options.before, options.after), + requests, + callerAnswers: options.answers, + requestToInput: + "not exposed by HTTP response; concurrent answers are not paired by order", + writeCapBytesPerSecond: writeCap, + timingReading: "taken under load", + latencyDistribution: null, + throughput: null, + ...options.extra, + }; + measure("Measurement", json(value)); + // A terminal record per submitted command is also the log-completeness check. + expect(completed).toHaveLength(options.answers.length); + expect(requests).toHaveLength(options.answers.length); + expect(new Set(own.map((record) => record.executionId)).size).toBe( + own.length, + ); + for (const request of requests) { + expect( + request.completionRecords.filter((record) => !record.willRetry), + ).toHaveLength(1); + for (const record of request.completionRecords) { + expect(typeof record.willRetry).toBe("boolean"); + expect(record).toHaveProperty("occInfo"); + if (record.willRetry) { + expect(record.occInfo).not.toBeNull(); + expect(typeof record.occInfo?.tableName).toBe("string"); + expect(record.error).toBeNull(); + } + if (record.error !== null || record.willRetry) { + expect(usageOf(record).databaseWriteDocuments).toBe(0); + expect(usageOf(record).databaseWriteBytes).toBe(0); + } + for (const count of Object.values(usageOf(record))) + expect(count).toBeGreaterThanOrEqual(0); + } + } + return value; +} +export async function measured( + world: ExperimentWorld, + args: OrderArgs[], + dataset: string, + path: string, + contention = 0, + extra: Record = {}, +) { + const backend = required(world.backend, "backend"); + const before = await stored(backend); + const mark = await backend.admin.logMark(); + const answers = await Promise.all(args.map((arg) => send(world, arg))); + // Close the interval with ordinary essential reads, one per tenant, before inspecting tables. + const summaries: Record = {}; + for (const tenant of new Set(args.map((arg) => arg.tenantId))) { + let cursor: string | null = null; + const keys: string[] = []; + for (;;) { + const page: FunctionReturnType = + await required(world.client, "client").query( + api.readModels.listOrderSummaries, + { + tenantId: tenant, + status: "placed" as const, + paginationOpts: { cursor, numItems: 100 }, + }, + ); + keys.push(...page.page.map((row) => row.key)); + if (page.isDone) break; + cursor = page.continueCursor; + } + summaries[tenant] = keys; + } + const records = await backend.admin.completionsSince( + mark, + (entries) => + finals(entries).length >= args.length && + entries.filter( + (record) => record.identifier === "readModels:listOrderSummaries", + ).length >= Object.keys(summaries).length, + 30000, + ); + const after = await stored(backend); + const scheduled = await schedules(backend); + const others = records.filter( + (record) => + !isOwn(record) && record.identifier !== "readModels:listOrderSummaries", + ); + const value = recordMeasurement(world, { + dataset, + lines: required(args[0], "first command").input.lines.length, + contention, + path, + answers, + records, + before, + after, + extra: { + scheduledFunctions: scheduled, + otherExecutions: json(others), + summaryKeys: summaries, + submittedInputs: json(args), + ...extra, + }, + }); + expect(others).toEqual([]); + expect(scheduled).toEqual({ parent: 0, orders: 0, inventory: 0 }); + expect(after.grants).toEqual(before.grants); + expect(after.generations).toEqual(before.generations); + for (const [index, answer] of answers.entries()) { + if (answer.kind !== "applied") continue; + const arg = required(args[index], "command"); + expect(summaries[arg.tenantId]).toContain(arg.input.orderId); + } + return { + value, + before, + after, + answers, + records: records.filter(isOwn) as ExecutionRecord[], + }; +} +export function expectUsage( + record: CompletionRecord, + [reads, writes, readBytes, writeBytes]: readonly [ + number, + number, + number, + number, + ], +) { + expect(usageOf(record)).toMatchObject({ + databaseReadDocuments: reads, + databaseWriteDocuments: writes, + databaseReadBytes: readBytes, + databaseWriteBytes: writeBytes, + databaseWriteIndexRows: required( + ({ 0: 0, 6: 25, 24: 97, 204: 817 } as Record)[writes], + "expected index writes", + ), + }); +} +export const healthyUsage = { + 1: [8, 6, 2943, 4271], + 10: [35, 24, 14454, 18410], + 100: [305, 204, 129564, 159800], +} as const; + +export type PlaceOrderMeasurement = ReturnType; diff --git a/tests/native/place-order-timing.test.ts b/tests/native/place-order-timing.test.ts new file mode 100644 index 0000000..3046300 --- /dev/null +++ b/tests/native/place-order-timing.test.ts @@ -0,0 +1,251 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { expect, test } from "vitest"; +import { performance } from "node:perf_hooks"; +import { api } from "../../example/convex/_generated/api.js"; +import { measure, required } from "../../harness/native.js"; +import { usageOf } from "./first-experiment-steps.js"; +import { stored } from "./place-order-attribution.js"; +import { + collect, + expectUsage, + finals, + healthyUsage, + json, + order, + pins, + recordMeasurement, + schedules, + seed, + send, + setup, + writeCap, + type Answer, +} from "./place-order-measurement.js"; +import { prepareCost, sampleCost, type CostWorld } from "./read-cost.js"; +const anchor = specTest({ + id: testAnchorId("test:application.first-experiment.timing"), + verifies: ref("spec:application.first-experiment"), +}); +void anchor; +const timing = test.skipIf(process.env.PLACE_ORDER_TIMINGS !== "1"); +const reading = + process.env.PLACE_ORDER_TIMING_LOAD === "quiet" + ? "quiet machine" + : "taken under load"; +const quantile = (values: number[], fraction: number) => + required( + [...values].sort((a, b) => a - b)[Math.ceil(values.length * fraction) - 1], + "percentile sample", + ); + +timing.each([1, 10, 100] as const)( + "native timing: PlaceOrder latency at %i lines", + async (lines) => { + const world = await setup(); + const backend = required(world.backend, "backend"); + const answers: Answer[] = []; + const records = []; + const discarded = []; + const before = await stored(backend); + for (let sample = 0; sample < 35; sample++) { + const args = order(lines, sample); + await seed(world, args); + const mark = await backend.admin.logMark(); + const answer = await send(world, args); + const completions = await collect(backend, mark, () => 1); + if (sample < 5) discarded.push(...completions); + else { + answers.push(answer); + records.push(...completions); + } + expect(answer).toMatchObject({ kind: "applied", replayed: false }); + expect(completions).toHaveLength(1); + expectUsage(required(completions[0], "completion"), healthyUsage[lines]); + } + const ms = finals(records).map((record) => record.executionTime * 1000); + const sorted = [...ms].sort((a, b) => a - b); + const medianMs = + (required(sorted[14], "lower middle") + + required(sorted[15], "upper middle")) / + 2; + const p95Ms = quantile(ms, 0.95); + const medianTargetMs = lines === 100 ? 400 : 100; + const p95TargetMs = lines === 100 ? 800 : 200; + const comparison = { + acceptedCommands: answers.length, + discardedCommands: 5, + medianMs, + p95Ms, + medianTargetMs, + p95TargetMs, + median: medianMs <= medianTargetMs ? "held" : "not held", + p95: p95Ms <= p95TargetMs ? "held" : "not held", + percentileMethod: "median averages middle pair; p95 uses nearest rank", + executionTimesMs: ms, + }; + const value = recordMeasurement(world, { + dataset: + "Five discarded then thirty measured applied commands; every order has fresh distinct stock received once with quantity two; four grants and one active generation; seven-character IDs; retained-row interval includes stock setup and discarded commands.", + lines, + contention: 0, + path: "latency", + answers, + records, + before, + after: await stored(backend), + extra: { + timingReading: reading, + latencyDistribution: comparison, + discardedCompletionRecords: json(discarded), + }, + }); + expect(value.engineReruns).toBe(0); + expect(answers).toHaveLength(30); + console.log( + JSON.stringify({ lines, timingReading: reading, ...comparison }), + ); + }, +); + +timing( + "native timing: eight callers place unrelated ten-line orders for ten seconds", + async () => { + const world = await setup(); + const backend = required(world.backend, "backend"); + const stock = Array.from({ length: 8 }, (_, caller) => order(10, caller)); + for (const args of stock) await seed(world, args, 100000); + const before = await stored(backend); + const mark = await backend.admin.logMark(); + const answers: Answer[] = []; + let done = false; + // The harness advances its cursor after every poll and refuses a batch of 1000 entries. + // Start polling before any caller sends, so retention never relies on the cell's final read. + const collecting = collect( + backend, + mark, + () => answers.length, + () => done, + 90000, + ).then( + (records) => ({ records }), + (error: unknown) => ({ error }), + ); + const started = performance.now(); + await Promise.all( + stock.map(async (template, caller) => { + let sequence = 0; + do { + const orderId = `o${String(caller * 100000 + sequence++).padStart(6, "0")}`; + answers.push( + await send(world, { + ...template, + requestKey: `k-${orderId}`, + input: { ...template.input, orderId }, + }), + ); + } while (performance.now() - started < 10000); + }), + ); + const elapsedSeconds = (performance.now() - started) / 1000; + done = true; + // Wake a poll that consumed the last completion before the last HTTP response arrived. + await required(world.client, "client").query( + api.readModels.listOrderSummaries, + { + tenantId: "t-1", + status: "placed", + paginationOpts: { cursor: null, numItems: 1 }, + }, + ); + const collected = await collecting; + if ("error" in collected) throw collected.error; + const applied = answers.filter( + (answer) => answer.kind === "applied" && !answer.replayed, + ).length; + const ordersPerSecond = applied / elapsedSeconds; + const throughput = { + callers: 8, + lines: 10, + elapsedSeconds, + completedOrders: applied, + ordersPerSecond, + targetOrdersPerSecond: 20, + target: ordersPerSecond >= 20 ? "held" : "not held", + writeCapBytesPerSecond: writeCap, + }; + const value = recordMeasurement(world, { + dataset: + "Eight callers, ten stock items per caller, no item touched by another caller; four grants in one tenant; quantity 100000 received once per stock item; each next order has a fresh fixed-length ID; closed-loop callers; no caller retries a rejected command.", + lines: 10, + contention: 8, + path: "throughput", + answers, + records: collected.records, + before, + after: await stored(backend), + extra: { + timingReading: reading, + throughput, + scheduledFunctions: await schedules(backend), + }, + }); + expect(elapsedSeconds).toBeGreaterThanOrEqual(10); + for (const record of finals(collected.records).filter( + (record) => record.error === null, + )) { + expect(usageOf(record).databaseReadDocuments).toBe(35); + expect(usageOf(record).databaseWriteDocuments).toBe(24); + expect(usageOf(record).databaseWriteBytes).toBe(18410); + expect(usageOf(record).databaseWriteIndexRows).toBe(97); + // The first order reads StockReceived; later orders read the larger StockAllocated tail. + expect([14454, 14614]).toContain(usageOf(record).databaseReadBytes); + } + expect( + finals(collected.records).filter( + (record) => + record.error === null && usageOf(record).databaseReadBytes === 14454, + ), + ).toHaveLength(8); + expect(value.rowsLeftBehind.generations.created).toBe(0); + console.log( + JSON.stringify({ + timingReading: reading, + ...throughput, + outcomes: value.outcomeCounts, + engineReruns: value.engineReruns, + }), + ); + }, + 120000, +); + +timing( + "native timing: component query cost inside a mutation against the same helper read", + async () => { + const world: CostWorld = {}; + await prepareCost(world, 100); + await sampleCost(world, "mutation"); + const medians = required(world.medians, "medians"); + const value = { + pins: pins(required(world.backend, "backend")), + composition: "fixture", + timingReading: reading, + readsPerSample: 100, + samplesPerPath: 9, + medianMs: { + helper: medians.helper * 1000, + nested: medians.nested * 1000, + component: medians.component * 1000, + }, + componentMinusHelperPerCallMs: (medians.component - medians.helper) * 10, + productionComponentCallCost: "not exposed", + quota: "not exposed", + }; + measure("componentCost", json(value)); + console.log(JSON.stringify(value)); + }, +); From b17337b6e928d5beb2c68967738d123bb124aa05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:05:13 +0200 Subject: [PATCH 006/155] design: the delegated decision on how Sc L2-9 is proven, its checked lean as the ruling Co-Authored-By: Claude Opus 5.5 --- design/decisions/register.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/design/decisions/register.json b/design/decisions/register.json index 200b8ca..6ca2bb8 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -257,7 +257,7 @@ "provisionalReading": null, "blocks": null, "advisor": "convex", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "delegated", "reason": "Rule 3: which example proves a row; the strict reading, an example that sends commands at once on the production composition, goes against no scenario of the doc and an edit takes it back. Doubt: the acceptance contract is the product advisor's Spec, the example is this family's.", @@ -271,9 +271,9 @@ }, "checkedBy": "gpt-6.1-sol, medium, 2026-10-02: every checked claim held", "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "advisor:convex", + "decidedOn": "2026-10-02", + "ruling": "Bind Sc L2-9 to the experiment's contention run on the production composition, 2, 8 and 32 PlaceOrder commands at once, and keep the present example as the row's code-path half." }, { "id": "OD-008", From 57d2d099c9753915d69f7f9e04d8810502b681c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:05:20 +0200 Subject: [PATCH 007/155] build: CancelOrder and release in the example domain The order decider gains cancel and OrderCancelled, refusing an order with no event (orderNotFound) and a cancelled one (orderAlreadyCancelled). The stock item decider gains release and AllocationReleased over its two totals, refusing a release above the quantity allocated (insufficientAllocation). Orders gains the cancel operation, Inventory the release operation, and the parent the CancelOrder use case, which calls Orders first and releases the lines Orders returned in the same mutation. The order DTO admits cancelled, so the order summary row CancelOrder writes says cancelled. Generated component files of the production composition from a fresh codegen. Spec: the example domain's model carries every name, shape, code and message, and owns an example space with six examples, each bound to a native test on the production composition; pure decider tests with the fold and rebuild, pure executor tests and convex-test tests beside them. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 17 +- design/specs/application.pack.sdp.md | 8 +- ...cel-duplicate-answered-from-receipt.sdp.md | 32 ++ ...ventory-example.cancel-placed-order.sdp.md | 34 +++ ...ample.cancel-second-context-rejects.sdp.md | 35 +++ ...ntory-example.cancel-twice-rejected.sdp.md | 34 +++ ...ample.cancel-unknown-order-rejected.sdp.md | 33 +++ ...xample.cancel-without-grant-refused.sdp.md | 32 ++ .../orders-inventory-example.sdp.md | 61 +++- .../convex/inventory/_generated/component.ts | 93 ++++++ example/convex/inventory/operations.ts | 21 ++ example/convex/inventory/streams.ts | 4 + example/convex/ordering.ts | 85 +++++- example/convex/orders/_generated/component.ts | 103 ++++++- example/convex/orders/operations.ts | 38 ++- example/convex/orders/streams.ts | 3 +- example/domain/order.ts | 58 +++- example/domain/stockItem.ts | 40 ++- tests/native/cancel-order-steps.ts | 277 ++++++++++++++++++ tests/native/order-use-case.ts | 10 +- ...el-duplicate-answered-from-receipt.test.ts | 40 +++ ...entory-example.cancel-placed-order.test.ts | 123 ++++++++ ...mple.cancel-second-context-rejects.test.ts | 61 ++++ ...tory-example.cancel-twice-rejected.test.ts | 46 +++ ...mple.cancel-unknown-order-rejected.test.ts | 54 ++++ ...ample.cancel-without-grant-refused.test.ts | 37 +++ tests/pure/example-domain.test.ts | 184 +++++++++++- tests/pure/example-use-cases.test.ts | 137 ++++++++- tests/simulator/example.test.ts | 257 +++++++++++++++- ...tion.renamed-handler-fails-build.test-d.ts | 5 +- 30 files changed, 1902 insertions(+), 60 deletions(-) create mode 100644 design/specs/application/orders-inventory-example.cancel-duplicate-answered-from-receipt.sdp.md create mode 100644 design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md create mode 100644 design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md create mode 100644 design/specs/application/orders-inventory-example.cancel-twice-rejected.sdp.md create mode 100644 design/specs/application/orders-inventory-example.cancel-unknown-order-rejected.sdp.md create mode 100644 design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md create mode 100644 tests/native/cancel-order-steps.ts create mode 100644 tests/native/orders-inventory-example.cancel-duplicate-answered-from-receipt.test.ts create mode 100644 tests/native/orders-inventory-example.cancel-placed-order.test.ts create mode 100644 tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts create mode 100644 tests/native/orders-inventory-example.cancel-twice-rejected.test.ts create mode 100644 tests/native/orders-inventory-example.cancel-unknown-order-rejected.test.ts create mode 100644 tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts diff --git a/design/README.md b/design/README.md index 96361e1..6611462 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -202 specs · 5 packs · 101 anchors → 308 nodes · 1097 edges (0 errors, 0 warnings) +208 specs · 5 packs · 107 anchors → 320 nodes · 1123 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -222,7 +222,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | | E-44 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | `BusinessFailurePolicy` (`carry` or `reject`) per context call, applied in the executor body through the pinned helper `carryOrReject` | | E-45 | [`application.read-models`](specs/application/read-models.sdp.md) | At most 8 component reads per parent query and 4 read-model rows written per command, counted as the rows step 9 inserts, replaces or deletes and checked after each `streams` entry is applied, above which step 9 throws a plain error | -| E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a provisional maximum of 100 lines; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` calls Orders first; the order allocation history view that Sc L2-6 builds | +| E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a provisional maximum of 100 lines; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` and `CancelOrder` call Orders first; the stock item keeps totals only and `CancelOrder` releases the cancelled order's own lines; `orderNotFound`, `orderAlreadyCancelled` and `insufficientAllocation` as `CancelOrder`'s rejections; the order allocation history view that Sc L2-6 builds | | E-47 | [`application.first-experiment`](specs/application/first-experiment.sdp.md) | The `Measurement` record, one per measured run, and how each count is captured: commits from the backend's function log, documents and bytes from the `usageStats` of the command's top-level completion record, the calls per context from a pure test of the use case's executor because the log holds no record of a component call inside a mutation, and a count the backend does not expose recorded as a gap; the documents one `PlaceOrder` reads, expected near N + 6 and counted 3N + 5 on the pinned backend, left for the measurement run to explain | | E-50 | [`obligations.obligation-module`](specs/obligations/obligation-module.sdp.md) | The module is a parent-side library plus two parent tables with a static handler registry, not a component | | E-51 | [`obligations.lifecycle-transitions`](specs/obligations/lifecycle-transitions.sdp.md), [`obligations.local-reaction-wrapper`](specs/obligations/local-reaction-wrapper.sdp.md), [`effects.external-effects`](specs/effects/external-effects.sdp.md) | A body's business failure settles `succeeded` with event evidence; a thrown rejection or a declined report settles `cancelled` with `businessRejection` evidence; backoff 1 s times 2 capped at 1 h; 8 attempts by default; a transient `writePaused` or `capacity` refusal defers the same attempt after `retryAfterMs` or 60 s without counting it | @@ -319,6 +319,19 @@ The Layer 2 design added two more for the same reason, and nine examples that ve Package A therefore has 79 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. +The example domain's second lifecycle command added six examples that verify no doc row, each a child of `spec:application.orders-inventory-example` under E-46: + +| Sc | Added example | Reason | +|---|---|---| +| none | `spec:application.orders-inventory-example.cancel-placed-order` | verifies that `CancelOrder` cancels a placed order, releases its lines and writes the summary's `cancelled` | +| none | `spec:application.orders-inventory-example.cancel-twice-rejected` | verifies that the order's own state refuses a second cancel | +| none | `spec:application.orders-inventory-example.cancel-duplicate-answered-from-receipt` | verifies that a repeated cancel request is answered from its receipt | +| none | `spec:application.orders-inventory-example.cancel-unknown-order-rejected` | verifies that a cancel of an order never placed is refused | +| none | `spec:application.orders-inventory-example.cancel-second-context-rejects` | verifies that a release Inventory refuses rolls back the cancel Orders recorded | +| none | `spec:application.orders-inventory-example.cancel-without-grant-refused` | verifies that `CancelOrder` requires its own permission | + +Package D therefore has 25 examples. + ## Integration notes What the integration pass changed after the five packages landed, and what it left for the owner or the plan. diff --git a/design/specs/application.pack.sdp.md b/design/specs/application.pack.sdp.md index 589e1c1..2dfb82c 100644 --- a/design/specs/application.pack.sdp.md +++ b/design/specs/application.pack.sdp.md @@ -27,6 +27,12 @@ specs: - spec:application.first-experiment.order-of-max-lines - spec:application.first-experiment.native-acceptance-production-configuration - spec:application.orders-inventory-example + - spec:application.orders-inventory-example.cancel-placed-order + - spec:application.orders-inventory-example.cancel-twice-rejected + - spec:application.orders-inventory-example.cancel-duplicate-answered-from-receipt + - spec:application.orders-inventory-example.cancel-unknown-order-rejected + - spec:application.orders-inventory-example.cancel-second-context-rejects + - spec:application.orders-inventory-example.cancel-without-grant-refused - spec:operations.baseline-operations - spec:operations.baseline-operations.broken-metrics-never-abort - spec:operations.baseline-operations.broken-audit-aborts @@ -46,4 +52,4 @@ modelRefs: Package D. Layer 2: parent use cases, read models and the projection contract, online rebuild with the generation registry and the write pause, restore, baseline operations, the cost-target constraints and the first experiment, with the Layer 2 scenarios and the all-layers audit scenario. -Membership order follows the plan's inventory with each parent's examples directly after it: parent use cases, read models, the projection contract, rebuild with its registry and gate, restore, the first experiment with the example domain, baseline operations, then the eight constraints. Thirty-seven Specs: nine application Specs, one operations Spec, eight constraints and nineteen examples. Three examples are siblings the plan's table did not list, `second-context-throws`, `committed-state-read-through-query` and `write-pause-rebuild-abort`, added because each doc row enumerates two cases. Five examples verify no doc row: `first-activation-makes-read-model-writable` and `command-without-generation-fails` verify the generation registry's first activation and its failure under E-8, `query-refused-before-disclosure` and `query-refused-without-grant` verify Law 5 for a parent query, and `list-pages-by-cursor` verifies a parent list under E-24. +Membership order follows the plan's inventory with each parent's examples directly after it: parent use cases, read models, the projection contract, rebuild with its registry and gate, restore, the first experiment with the example domain, baseline operations, then the eight constraints. Forty-three Specs: nine application Specs, one operations Spec, eight constraints and twenty-five examples. Three examples are siblings the plan's table did not list, `second-context-throws`, `committed-state-read-through-query` and `write-pause-rebuild-abort`, added because each doc row enumerates two cases. Five examples verify no doc row: `first-activation-makes-read-model-writable` and `command-without-generation-fails` verify the generation registry's first activation and its failure under E-8, `query-refused-before-disclosure` and `query-refused-without-grant` verify Law 5 for a parent query, and `list-pages-by-cursor` verifies a parent list under E-24. Six more verify no doc row: the `cancel-` examples of the example domain verify `CancelOrder` under E-46, the second lifecycle command, its rejections, its receipt and its grant. diff --git a/design/specs/application/orders-inventory-example.cancel-duplicate-answered-from-receipt.sdp.md b/design/specs/application/orders-inventory-example.cancel-duplicate-answered-from-receipt.sdp.md new file mode 100644 index 0000000..1e789ec --- /dev/null +++ b/design/specs/application/orders-inventory-example.cancel-duplicate-answered-from-receipt.sdp.md @@ -0,0 +1,32 @@ +--- +id: spec:application.orders-inventory-example.cancel-duplicate-answered-from-receipt +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.orders-inventory-example + verifies: spec:application.orders-inventory-example +--- +# A repeated cancel request is answered from its receipt + +E-46 · native tier · production composition · no acceptance row; the example verifies that `CancelOrder` is answered from its receipt when its request is repeated. + +## Intent + +- outcome: The same request under the same key is a duplicate, answered from the first cancel's receipt without running again, and nothing is stored. (E-46, D6) + +```gwt +Given {onHand: 5} units of a stock item received through ReceiveStock +And an order placed for {ordered: 3} of them over two lines +And the order {before: "is cancelled under request key k-1"} +When a caller {grant: "holding"} the permission orders.cancel sends CancelOrder for {target: "that order"} under request key {key: "k-1"} +Then the caller receives {response: "the receipt's answer"} +And the stock item's quantity allocated is {allocated: 0} +And the order summary's status is {status: "cancelled"} +And the number of documents the command wrote is {written: 0} +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- The test asserts that the response is `{ kind: "applied", replayed: true, result: null }` with the first cancel's `operationId` and versions, and that the receipts table holds one receipt for `k-1`. diff --git a/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md b/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md new file mode 100644 index 0000000..9df1416 --- /dev/null +++ b/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md @@ -0,0 +1,34 @@ +--- +id: spec:application.orders-inventory-example.cancel-placed-order +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.orders-inventory-example + verifies: spec:application.orders-inventory-example +--- +# A placed order is cancelled + +E-46 · native tier · production composition · no acceptance row; the example verifies `CancelOrder`, the example domain's second lifecycle command under E-46. + +## Intent + +- outcome: The order is cancelled, its stock item's totals are back to what they were before it was placed, and its summary row says cancelled, in one commit. (E-46, D10) + +```gwt +Given {onHand: 5} units of a stock item received through ReceiveStock +And an order placed for {ordered: 3} of them over two lines +When a caller {grant: "holding"} the permission orders.cancel sends CancelOrder for {target: "that order"} under request key {key: "k-1"} +Then the caller receives {response: "the result"} +And the stock item's quantity allocated is {allocated: 0} +And the order summary's status is {status: "cancelled"} +And the number of documents the command wrote is {written: 6} +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- The order is `order-1` with two lines on the stock item `sku-1`, of 2 and 1 units; the test asserts that the stock item's state after the cancel equals its state before `PlaceOrder`, `{ onHand: 5, allocated: 0 }`. +- The test asserts the response `{ kind: "applied", replayed: false, result: { orderId: "order-1", released: [{ stockItemId: "sku-1", quantity: 3 }] } }` with versions naming the order at version 2 and the stock item at version 3, Orders first. +- The six documents are the receipt, the order's state and its `OrderCancelled` event, the stock item's state and its one `AllocationReleased` event of quantity 3, and the summary row, whose `sourceVersions` name the order at version 2; the parent's `getOrder` answers the order with status `cancelled`, its lines, total and `placedAt` unchanged. +- The test reads the function log and asserts that the command ran as one top-level mutation, and that the order summary list for `cancelled` holds the order and the list for `placed` does not. diff --git a/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md b/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md new file mode 100644 index 0000000..e637665 --- /dev/null +++ b/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md @@ -0,0 +1,35 @@ +--- +id: spec:application.orders-inventory-example.cancel-second-context-rejects +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.orders-inventory-example + verifies: spec:application.orders-inventory-example +--- +# Orders cancels, then Inventory refuses the release + +E-46 · native tier · production composition · no acceptance row; the example verifies that a release Inventory refuses rolls back the cancel Orders recorded, which is Sc L2-1's rule on the second lifecycle command. + +## Intent + +- outcome: Everything rolls back; the caller gets Inventory's rejection; the order stays placed and nothing is stored. (E-46, D1, D7, Sc L2-1) + +```gwt +Given {onHand: 5} units of a stock item received through ReceiveStock +And an order placed for {ordered: 3} of them over two lines +And the order {before: "has its units released by Inventory alone"} +When a caller {grant: "holding"} the permission orders.cancel sends CancelOrder for {target: "that order"} under request key {key: "k-1"} +Then the caller receives {response: "the rejection"} +And the rejection code is {code: "insufficientAllocation"} +And the stock item's quantity allocated is {allocated: 0} +And the order summary's status is {status: "placed"} +And the number of documents the command wrote is {written: 0} +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- No public command releases stock without cancelling its order, so the setup calls the Inventory context's release operation for the order's 3 units directly, with admin access, as a sanctioned operation of that context; the stock item's state stays the fold of its events, and the two contexts now disagree about the order. +- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "insufficientAllocation", commandType: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 } }`, which only a release planned from the lines the Orders call returned can carry. +- The test reads the function log and asserts that the request ran one top-level mutation, recorded as failed with no document written, and that the order's stream stays at version 1 with status `placed` and one event. diff --git a/design/specs/application/orders-inventory-example.cancel-twice-rejected.sdp.md b/design/specs/application/orders-inventory-example.cancel-twice-rejected.sdp.md new file mode 100644 index 0000000..f6ea881 --- /dev/null +++ b/design/specs/application/orders-inventory-example.cancel-twice-rejected.sdp.md @@ -0,0 +1,34 @@ +--- +id: spec:application.orders-inventory-example.cancel-twice-rejected +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.orders-inventory-example + verifies: spec:application.orders-inventory-example +--- +# A second cancel of one order is rejected + +E-46 · native tier · production composition · no acceptance row; the example verifies that the order's own state refuses a second cancel. + +## Intent + +- outcome: The second cancel, under a new request key, is rejected by the order's state before Inventory decides, and nothing is stored. (E-46, D4, D7) + +```gwt +Given {onHand: 5} units of a stock item received through ReceiveStock +And an order placed for {ordered: 3} of them over two lines +And the order {before: "is cancelled under request key k-1"} +When a caller {grant: "holding"} the permission orders.cancel sends CancelOrder for {target: "that order"} under request key {key: "k-2"} +Then the caller receives {response: "the rejection"} +And the rejection code is {code: "orderAlreadyCancelled"} +And the stock item's quantity allocated is {allocated: 0} +And the order summary's status is {status: "cancelled"} +And the number of documents the command wrote is {written: 0} +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "orderAlreadyCancelled", commandType: "CancelOrder", message: "The order is already cancelled" }`, and that no receipt exists for `k-2`. +- The stock item's state stays `{ onHand: 5, allocated: 0 }`; had the second cancel reached Inventory, its release of 3 would have been refused `insufficientAllocation`, so the code `orderAlreadyCancelled` shows that Orders refused it first. diff --git a/design/specs/application/orders-inventory-example.cancel-unknown-order-rejected.sdp.md b/design/specs/application/orders-inventory-example.cancel-unknown-order-rejected.sdp.md new file mode 100644 index 0000000..6d6fea1 --- /dev/null +++ b/design/specs/application/orders-inventory-example.cancel-unknown-order-rejected.sdp.md @@ -0,0 +1,33 @@ +--- +id: spec:application.orders-inventory-example.cancel-unknown-order-rejected +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.orders-inventory-example + verifies: spec:application.orders-inventory-example +--- +# A cancel of an order never placed is rejected + +E-46 · native tier · production composition · no acceptance row; the example verifies that the order decider refuses a cancel of an order with no event. + +## Intent + +- outcome: A cancel of an order ID that has no stream is rejected by the order decider, no stream is created for it, and nothing is stored. (E-46, D4, D7) + +```gwt +Given {onHand: 5} units of a stock item received through ReceiveStock +And an order placed for {ordered: 3} of them over two lines +When a caller {grant: "holding"} the permission orders.cancel sends CancelOrder for {target: "an order never placed"} under request key {key: "k-1"} +Then the caller receives {response: "the rejection"} +And the rejection code is {code: "orderNotFound"} +And the stock item's quantity allocated is {allocated: 3} +And the order summary's status is {status: "placed"} +And the number of documents the command wrote is {written: 0} +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- The order never placed is `order-2`; the test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "orderNotFound", commandType: "CancelOrder", message: "The order does not exist" }`, that the Orders context holds no stream row and no event for `order-2`, and that the parent's `getOrder` answers null for it. +- The order summary's status is that of the placed order `order-1`, the one row the table holds. diff --git a/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md b/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md new file mode 100644 index 0000000..f781254 --- /dev/null +++ b/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md @@ -0,0 +1,32 @@ +--- +id: spec:application.orders-inventory-example.cancel-without-grant-refused +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.orders-inventory-example + verifies: spec:application.orders-inventory-example +--- +# A caller without the cancel grant is refused + +E-46 · native tier · production composition · no acceptance row; the example verifies that `CancelOrder` requires its own permission, under Law 5 and D11. + +## Intent + +- outcome: A caller who holds every grant of the composition except `orders.cancel` is refused before any execution, and nothing is stored. (E-46, D11, Law 5) + +```gwt +Given {onHand: 5} units of a stock item received through ReceiveStock +And an order placed for {ordered: 3} of them over two lines +When a caller {grant: "lacking"} the permission orders.cancel sends CancelOrder for {target: "that order"} under request key {key: "k-1"} +Then the caller receives {response: "the rejection"} +And the rejection code is {code: "forbidden"} +And the stock item's quantity allocated is {allocated: 3} +And the order summary's status is {status: "placed"} +And the number of documents the command wrote is {written: 0} +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- The refused caller is a second user of the fixture issuer holding `orders.place`, `inventory.receive`, `orders.read` and `inventory.read` in the tenant; the test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "forbidden", commandType: "CancelOrder", message: "The caller may not run CancelOrder in this tenant", details: { reason: "no_grant" } }`. diff --git a/design/specs/application/orders-inventory-example.sdp.md b/design/specs/application/orders-inventory-example.sdp.md index aac1022..c72f3ad 100644 --- a/design/specs/application/orders-inventory-example.sdp.md +++ b/design/specs/application/orders-inventory-example.sdp.md @@ -7,15 +7,19 @@ relations: refines: spec:application.first-experiment decidedBy: - spec:decisions.d04-four-outcomes + - spec:decisions.d07-rejections-thrown-not-stored - spec:decisions.d10-contexts-meet-in-parent-use-cases constrainedBy: - spec:laws.law01-sanctioned-writes-only + - spec:laws.law09-no-invariant-on-late-read-model --- # Orders and Inventory, the example domain -Story · Detail: full · Traces: First experiment, Law 1, D4, D9, D10, D16, Sc L1-4, Sc L1-11, Sc L2-3, Sc L2-6, E-2, E-9, E-39, E-46. +Story · Detail: full · Traces: First experiment, Law 1, Law 9, D3, D4, D6, D7, D9, D10, D16, Sc L1-4, Sc L1-11, Sc L2-3, Sc L2-6, E-2, E-9, E-38, E-39, E-46. -The first experiment's domain has two bounded contexts, Orders and Inventory, one composed use case that makes them meet, one more lifecycle command, one command that creates stock, and one essential summary. The terms below are the ones the experiment's code, tests and measurements use. Where the doc names a thing, its word is kept; the second lifecycle command, the command that creates stock, the summary and the rejection reason are the design's choices under E-46. `StartCheckout` from D16 is not part of the experiment; it is the Layer 4 contrast to atomic `PlaceOrder`. +The first experiment's domain has two bounded contexts, Orders and Inventory, one composed use case that makes them meet, one more lifecycle command, one command that creates stock, and one essential summary. The terms below are the ones the experiment's code, tests and measurements use. Where the doc names a thing, its word is kept; the second lifecycle command, the command that creates stock, the summary and the rejection reasons are the design's choices under E-46. `StartCheckout` from D16 is not part of the experiment; it is the Layer 4 contrast to atomic `PlaceOrder`. + +The second lifecycle command is `CancelOrder`. The stock item keeps two totals and no allocation per order, so a release takes its quantities from the cancelled order's own lines, which Orders returns in the same mutation, and a second cancel is refused by the order's own state; no stock item remembers an order. ## Intent @@ -24,35 +28,60 @@ The first experiment's domain has two bounded contexts, Orders and Inventory, on ### Open questions -- [non-blocking] Extension E-46: the second lifecycle command, the essential summary, the command that creates stock, the unit price an order line carries, the rejection reason for short stock, the stream budgets of the two stream types, the order of the two context calls and the history-dependent view Sc L2-6 builds are choices the doc leaves open; the owner may substitute others, and the experiment's terms move with them; the owner has not ruled on any of them (E-46, E-2, E-9, First experiment) +- [non-blocking] Extension E-46: the second lifecycle command, the essential summary, the command that creates stock, the unit price an order line carries, the rejection reasons, the stream budgets of the two stream types, the order of the two context calls and the history-dependent view Sc L2-6 builds are choices the doc leaves open; two of them are decided, that the stock item keeps totals only and `release` takes its quantities from the cancelled order's own lines, and that the order allocation history view stays in the experiment; the owner may substitute the others, and the experiment's terms move with them (E-46, E-2, E-9, First experiment) ## Model -- **Orders context** — The bounded context that owns orders as streams, one document per order, with its own journal and the `PlaceOrder` and `CancelOrder` decisions (First experiment, D2). +- **Orders context** — The bounded context that owns orders as streams, one document per order, with its own journal and the place and cancel decisions (First experiment, D2). - **Inventory context** — The bounded context that owns stock items as streams, one document per stock item, with its own journal and the receive, allocate and release decisions (First experiment, D2, E-46). -- **order** — A stream in the Orders context identified by an application order ID; its state holds its lines, its status, its total and `placedAt`, the time it was placed, taken from `OrderPlaced`'s `occurredAt`, under the default single mapping with the default `budgetBytes` of 256 KiB, which holds the maximum order (First experiment, D2, E-2). -- **order line** — One requested stock item, quantity and unit price inside an order, the price in whole minor units as the caller states it; the number of lines is the size the experiment measures, and the order's total is the sum of quantity times unit price over its lines (First experiment, Sc L2-3, E-46). -- **stock item** — A stream in the Inventory context identified by an application stock item ID; its state holds the quantity on hand and the quantity allocated, the quantity available being the first less the second, under `mapping: { kind: "single", budgetBytes: 16 * 1024 }`, a budget declared small because the state is two quantities, so that the allocation operation admits the maximum order's streams in one call under the byte-derived bound (First experiment, D2, E-2, E-46). -- **allocation** — The Inventory context's reservation of quantity on a stock item for one order, released when the order is cancelled (First experiment, E-46). +- **order** — A stream in the Orders context identified by an application order ID; its state is `{ status: "none" | "placed" | "cancelled"; lines: OrderLine[]; total: number; placedAt: number | null }`, where `none` is a stream with no event and `placedAt` is the time it was placed, taken from `OrderPlaced`'s `occurredAt`, under the default single mapping with the default `budgetBytes` of 256 KiB, which holds the maximum order (First experiment, D2, E-2, E-46). +- **order line** — One requested stock item, quantity and unit price inside an order, `{ stockItemId: string; quantity: number; unitPrice: number }`, the price in whole minor units as the caller states it; the number of lines is the size the experiment measures, and the order's total is the sum of quantity times unit price over its lines (First experiment, Sc L2-3, E-46). +- **order DTO** — What the Orders context returns for an order that has an event, `{ orderId: string; status: "placed" | "cancelled"; lines: OrderLine[]; total: number; placedAt: number; version: StreamVersion }`; an order with no event has no DTO, and a cancelled order keeps its lines, total and `placedAt` (D10, E-46). +- **order invariant** — `aPlacedOrderHasALineATimeAndItsTotal`: an order that is placed or cancelled has at least one line, a `placedAt` and the total of its lines (D3, E-46). +- **stock item** — A stream in the Inventory context identified by an application stock item ID; its state is two totals, `{ onHand: number; allocated: number }`, the quantity on hand and the quantity allocated, and no allocation per order, the quantity available being the first less the second, under `mapping: { kind: "single", budgetBytes: 16 * 1024 }`, a budget declared small because the state is two quantities, so that the allocate and release operations admit the maximum order's streams in one call under the byte-derived bound (First experiment, D2, E-2, E-46). +- **stock item invariant** — `allocatedIsAWholeNumberBetweenZeroAndOnHand`: both totals are safe whole numbers and the quantity allocated is at least 0 and at most the quantity on hand (D3, E-46). +- **allocation** — The Inventory context's reservation of quantity on a stock item for one order; `StockAllocated` names the order, the stock item's state keeps only the summed total, and the allocation is released when the order is cancelled (First experiment, E-46). - **allocate operation** — The Inventory context's list-shaped operation `allocate({ orderId, lines })`, planning one stock item stream per distinct stock item with the quantities of its lines summed, and declaring `maxStreams` equal to the maximum order size, so its byte bound at the stock item's budget admits the maximum (D10, E-2, E-46, Sc L2-3). +- **release operation** — The Inventory context's list-shaped operation `release({ orderId, lines: [{ stockItemId, quantity }] })`, planning one `release` stream command per distinct stock item with the quantities of its lines summed, in the order stock items first appear, with no expected version and `maxStreams` equal to the maximum order size; it returns `{ lines: [{ stockItemId, quantity }] }`, one entry per stock item released (D10, E-2, E-46). - **place operation** — The Orders context's list-shaped operation `place({ orders: [{ orderId, lines }] })`, planning one order stream per order at expected version 0, so a second place of the same order ID is answered `entityExists` before its decider runs (D10, E-46, Sc L1-4). +- **cancel operation** — The Orders context's list-shaped operation `cancel({ orders: [{ orderId }] })`, planning one `cancel` stream command per order with no expected version, so the order's own state decides it, and declaring `maxStreams` 32 as the place operation does; it returns `{ orders: [{ orderId, lines }] }`, the lines of each cancelled order taken from its DTO (D10, E-46). - **receive operation** — The Inventory context's list-shaped operation `receive({ items })`, planning one stock item stream per distinct stock item, which creates the stream of a stock item that has none and adds the quantity to the quantity on hand; it is the only way a stock item comes to exist (Law 1, D10, E-46). - **PlaceOrder** — The composed use case that records an order in Orders and allocates every line in Inventory in one mutation, with one receipt and one outcome, calling Orders first because it creates the order, so a UI double submit is answered `entityExists` before Inventory decides, and Inventory second; it rejects when stock is short, and an order with no line is refused `invalidInput` before either context is called (D10, D4, E-39, Sc L1-4). - **ReceiveStock** — The command that receives stock, one call to the Inventory context's receive operation with one receipt and one outcome; every stock item of the experiment, a test's and a measurement's included, is created through it (Law 1, D10, E-46). -- **CancelOrder** — The second lifecycle command: the composed use case that cancels an order in Orders and releases its allocations in Inventory in one mutation (E-46, First experiment). -- **permissions** — `orders.place`, which `PlaceOrder` requires, `inventory.receive`, which `ReceiveStock` requires, and the read permissions `orders.read`, which the parent queries over orders and the order summary require, and `inventory.read` (D11, E-37, E-46). -- **OrderPlaced** — The Orders event recorded by `PlaceOrder`, carrying the accepted lines and the total, bounded by the events-stay-small constraint (First experiment, D2). -- **OrderCancelled** — The Orders event recorded by `CancelOrder` (E-46). +- **CancelOrder** — The second lifecycle command: the composed use case `cancelOrder`, input `{ orderId: string }`, that calls the Orders context's cancel operation once with the one order and then the Inventory context's release operation once with the order ID and the stock item and quantity of each line Orders returned for that order, in one mutation with one receipt and one outcome, and answers `{ orderId: string; released: { stockItemId: string; quantity: number }[] }`, the release operation's lines; Orders comes first because the order's state refuses a cancel before Inventory decides, and the quantities are read from what Orders returned in this mutation, never from a read model (D10, D4, Law 9, E-39, E-46). +- **CancelOrder declaration** — `name: "CancelOrder"`, `contractVersion: 1`, `permission: { permission: "orders.cancel" }`, `writes` the Orders `order` and the Inventory `stockItem`, `readModels` the order summary from the Orders `order`, `rejections: ["orderNotFound", "orderAlreadyCancelled", "insufficientAllocation"]` and no `bounds`, because the input holds no list, the streams the command writes are its one order and the stock items of that order's lines, at most the 100 `PlaceOrder` admitted, and the order ID is bounded at 256 bytes as every stream ID is (D6, D10, E-38, E-46). +- **permissions** — `orders.place`, which `PlaceOrder` requires, `orders.cancel`, which `CancelOrder` requires, `inventory.receive`, which `ReceiveStock` requires, and the read permissions `orders.read`, which the parent queries over orders and the order summary require, and `inventory.read` (D11, E-37, E-46). +- **OrderPlaced** — The Orders event recorded by `PlaceOrder`, carrying the accepted lines and the total, `{ lines, total }`, bounded by the events-stay-small constraint (First experiment, D2). +- **OrderCancelled** — The Orders event recorded by `CancelOrder` on a placed order, with the empty payload `{}` and the time in `occurredAt`; folding it sets the order's status to `cancelled` and keeps its lines, total and `placedAt` (E-46, D2). - **StockReceived** — The Inventory event recorded per stock item when `ReceiveStock` receives a quantity (E-46). - **StockAllocated** — The Inventory event recorded per stock item when `PlaceOrder` allocates quantity to an order; it carries the order ID and the quantity (First experiment, D2). -- **AllocationReleased** — The Inventory event recorded per stock item when `CancelOrder` releases a line (E-46). -- **short stock rejection** — The rejection `insufficientStock` that the Inventory context throws when a stock item's available quantity is below the quantity an order asks for, carrying the requested and the available quantity; `PlaceOrder` lists the code, nothing commits, and an order that waits for stock would be a different command (D4, D7, E-46). +- **AllocationReleased** — The Inventory event recorded per stock item when `CancelOrder` releases the cancelled order's lines on it, carrying the order ID and the summed quantity, `{ orderId, quantity }`; folding it subtracts the quantity from the quantity allocated and leaves the quantity on hand (E-46, D2). +- **short stock rejection** — The rejection `insufficientStock` that the Inventory context throws when a stock item's available quantity is below the quantity an order asks for, with the message "Cannot allocate N when M are available" and the details `{ requested, available }`; `PlaceOrder` lists the code, nothing commits, and an order that waits for stock would be a different command (D4, D7, E-46). +- **unknown order rejection** — The rejection `orderNotFound` that the order decider throws for a cancel of an order with no event, with the message "The order does not exist" and no details; `CancelOrder` lists the code, and nothing commits (D4, D7, E-46). +- **second cancel rejection** — The rejection `orderAlreadyCancelled` that the order decider throws for a cancel of a cancelled order, with the message "The order is already cancelled" and no details, so a second cancel never reaches Inventory and no stock item has to remember an order; `CancelOrder` lists the code, and nothing commits (D4, D7, E-46). +- **release above allocation rejection** — The rejection `insufficientAllocation` that the stock item decider throws for a release larger than the stock item's quantity allocated, with the message "Cannot release N when M are allocated" and the details `{ requested, allocated }`; a cancel of a placed order does not ask for one, because `PlaceOrder` allocated every line in the mutation that recorded the order and one order is cancelled once, and if the two contexts ever disagree `CancelOrder` lists the code and nothing commits (D4, D7, E-46). +- **invalid quantity rejection** — The rejection `invalidQuantity`. The order decider throws it for a place whose line has a quantity below 1 or not whole or a unit price below 0 or not whole, and for lines whose total passes the largest safe whole number; the stock item decider throws it for a receive, allocate or release of a quantity that is not a positive whole number, with the message "A quantity must be a positive whole number, not N" and the details `{ quantity }` (D4, D7, E-46). - **stock contention** — Two `PlaceOrder` commands for the same stock item sent concurrently; on the native backend both are applied when the stock covers both, and one is applied and one is rejected `insufficientStock` when it covers only one, and the experiment measures the retry cost of the loser (Sc L1-11, First experiment). -- **order summary** — The essential summary: a per-entity read model of the parent with the order's status, line count, total and the time it was placed, written from the order's DTO by every command that writes the order, and listed by status (E-46, D8). +- **order summary** — The essential summary: a per-entity read model of the parent with the order's status, line count, total and the time it was placed, `{ orderId, status: "placed" | "cancelled", lineCount, total, placedAt }`, written from the order's DTO by every command that writes the order, so `PlaceOrder` inserts the row with `placed` and `CancelOrder` replaces it with `cancelled`, and listed by status (E-46, D8). - **order allocation history** — A history-dependent cross-stream read model of the parent keyed by order ID, folded as a `HistoryProjection` from `OrderPlaced` in Orders and `StockAllocated` and `AllocationReleased` across the stock item streams the order touched, holding per line when it was allocated and released; it is bound once per source stream type, rebuilds only under the write pause, and exists in the experiment for Sc L2-6, not as an essential read (E-9, E-46, D9, Sc L2-6). - **journal inspection** — The context queries that return a stream's events and its current version for a reviewer, used by the experiment and the restore checks (First experiment, D2). -- **maximum order size** — The largest number of lines `PlaceOrder` accepts, a product decision under OQ3, provisionally 100 for measurement (OQ3, E-46). +- **maximum order size** — The largest number of lines `PlaceOrder` accepts, 100, the bound its declaration carries: the example's promise and no platform limit, chosen by the command's author under the adapter's stream and byte ceilings (OQ3, E-46). + +## Example space + +```gwt-vocabulary +Given {onHand:number} units of a stock item received through ReceiveStock +And an order placed for {ordered:number} of them over two lines +And the order {before:"is cancelled under request key k-1"|"has its units released by Inventory alone"} +When a caller {grant:"holding"|"lacking"} the permission orders.cancel sends CancelOrder for {target:"that order"|"an order never placed"} under request key {key:string} +Then the caller receives {response:"the result"|"the receipt's answer"|"the rejection"} +And the rejection code is {code:"orderAlreadyCancelled"|"orderNotFound"|"insufficientAllocation"|"forbidden"} +And the stock item's quantity allocated is {allocated:number} +And the order summary's status is {status:"placed"|"cancelled"} +And the number of documents the command wrote is {written:number} +``` ## Verification — reviewed - A reviewer confirms that the experiment's code, tests and measurements use these terms and no synonyms, and that `PlaceOrder`, `CancelOrder` and `ReceiveStock` are the only public commands of the experiment. +- A reviewer confirms that the stock item's state holds the two totals and nothing per order, and that `CancelOrder` takes the quantities it releases from the lines the Orders context returned in the same mutation. diff --git a/example/convex/inventory/_generated/component.ts b/example/convex/inventory/_generated/component.ts index 238bb22..41388dd 100644 --- a/example/convex/inventory/_generated/component.ts +++ b/example/convex/inventory/_generated/component.ts @@ -207,6 +207,99 @@ export type ComponentApi = }, Name >; + release: FunctionReference< + "mutation", + "internal", + { + actor: { + delegationRef?: string; + id: string; + issuer?: string; + kind: "human" | "service" | "agent" | "reviewer" | "operator"; + onBehalfOf?: { + id: string; + kind: "human" | "service" | "agent" | "reviewer" | "operator"; + }; + }; + facts?: Record; + input: { + lines: Array<{ quantity: number; stockItemId: string }>; + orderId: string; + }; + operation: { + causedBy: + | { commandType: string; kind: "command" } + | { + contextId: string; + eventId: string; + kind: "event"; + tenantId: string; + } + | { kind: "migration"; migrationName: string }; + correlationId?: string; + operationId: string; + }; + tenantId: string; + }, + { + kind: "applied" | "businessFailure"; + result: { lines: Array<{ quantity: number; stockItemId: string }> }; + streams: Array<{ + appended: number; + created: boolean; + dto: any; + events: Array<{ + actor: { + delegationRef?: string; + id: string; + issuer?: string; + kind: "human" | "service" | "agent" | "reviewer" | "operator"; + onBehalfOf?: { + id: string; + kind: "human" | "service" | "agent" | "reviewer" | "operator"; + }; + }; + causedBy: + | { commandType: string; kind: "command" } + | { + contextId: string; + eventId: string; + kind: "event"; + tenantId: string; + } + | { kind: "migration"; migrationName: string }; + contextId: string; + correlationId?: string; + eventId: string; + eventSchemaVersion: number; + eventType: string; + occurredAt?: number; + operationId: string; + payload: any; + recordedAt: number; + streamId: string; + streamType: string; + streamVersion: number; + tenantId: string; + }>; + version: { + contextId: string; + streamId: string; + streamType: string; + tenantId: string; + version: number; + }; + }>; + versions: Array<{ + contextId: string; + streamId: string; + streamType: string; + tenantId: string; + version: number; + }>; + }, + Name + >; }; queries: { stockItem: { diff --git a/example/convex/inventory/operations.ts b/example/convex/inventory/operations.ts index c4ce046..4ba25a8 100644 --- a/example/convex/inventory/operations.ts +++ b/example/convex/inventory/operations.ts @@ -80,3 +80,24 @@ export const allocate = defineOperation< combine: (results) => ({ lines: quantitiesOf(results) }), maxStreams, }); +const releaseInput = { orderId: v.string(), lines: quantities }; +const releaseResult = v.object({ lines: quantities }); +// Releases each stock item's summed quantity from the quantity allocated. The quantities are the +// cancelled order's own lines, so the stock item keeps no allocation per order. +export const release = defineOperation< + ObjectType, + Infer +>(journal, { + name: "release", + streams: [stockItemStream], + input: releaseInput, + returns: releaseResult, + plan: ({ orderId, lines }) => + plans(lines, (quantity) => ({ + commandType: "release", + orderId, + quantity, + })), + combine: (results) => ({ lines: quantitiesOf(results) }), + maxStreams, +}); diff --git a/example/convex/inventory/streams.ts b/example/convex/inventory/streams.ts index 85c2c75..5d80170 100644 --- a/example/convex/inventory/streams.ts +++ b/example/convex/inventory/streams.ts @@ -37,6 +37,10 @@ export const stockItemStream: StreamRegistration< eventValidators: { StockReceived: v.object({ quantity: v.number() }), StockAllocated: v.object({ orderId: v.string(), quantity: v.number() }), + AllocationReleased: v.object({ + orderId: v.string(), + quantity: v.number(), + }), }, dto: stockItemDtoValidator, toDto: (state, meta): StockItemDto => ({ diff --git a/example/convex/ordering.ts b/example/convex/ordering.ts index b1861ed..5ba847f 100644 --- a/example/convex/ordering.ts +++ b/example/convex/ordering.ts @@ -1,8 +1,10 @@ -// PlaceOrder, the use case that makes the two contexts meet: it records the order in Orders and then -// allocates every line in Inventory, in one mutation with one receipt and one outcome. Orders comes -// first because it creates the order, so a second submit of the same order is answered entityExists -// before Inventory decides. A rejection or a throw in Inventory rolls back the Orders call with the -// whole mutation, so the executor catches nothing. +// The ordering flow's two use cases, where the two contexts meet. PlaceOrder records the order in +// Orders and then allocates every line in Inventory; CancelOrder cancels the order in Orders and then +// releases its lines in Inventory. Each runs in one mutation with one receipt and one outcome, and +// calls Orders first: PlaceOrder because Orders creates the order, so a second submit of the same +// order is answered entityExists before Inventory decides, and CancelOrder because the order's own +// state refuses a cancel before Inventory decides. A rejection or a throw in Inventory rolls back the +// Orders call with the whole mutation, so neither executor catches anything. import { v, type Infer } from "convex/values"; import { internalCommand, @@ -13,9 +15,11 @@ import { components } from "./_generated/api.js"; import { orderSummary } from "./orderSummary.js"; import { orderLineValidator } from "./orders/streams.js"; export const placeOrderPermission = "orders.place"; -// The largest order PlaceOrder accepts, provisional for measurement. +export const cancelOrderPermission = "orders.cancel"; +// The largest order PlaceOrder accepts: the example's promise, chosen under the adapter's ceilings. export const maxOrderLines = 100; const orderSource = { contextId: "orders", streamType: "order" }; +const stockItemSource = { contextId: "inventory", streamType: "stockItem" }; const placeOrderInput = v.object({ orderId: v.string(), lines: v.array(orderLineValidator), @@ -36,7 +40,7 @@ export const placeOrderDeclaration: CommandDeclaration< lines.length === 0 ? { message: "An order needs at least one line" } : null, output: placeOrderResult, permission: { permission: placeOrderPermission }, - writes: [orderSource, { contextId: "inventory", streamType: "stockItem" }], + writes: [orderSource, stockItemSource], readModels: [{ readModel: orderSummary, source: orderSource }], rejections: ["insufficientStock", "invalidQuantity"], bounds: { maxItems: maxOrderLines }, @@ -83,3 +87,70 @@ export const placeOrderDeclaration: CommandDeclaration< }; export const placeOrder = publicCommand(placeOrderDeclaration); export const placeOrderInternal = internalCommand(placeOrderDeclaration); +const cancelOrderInput = v.object({ orderId: v.string() }); +const quantity = v.object({ stockItemId: v.string(), quantity: v.number() }); +const cancelOrderResult = v.object({ + orderId: v.string(), + released: v.array(quantity), +}); +// The input holds no list, so it declares no bounds: the streams it writes are its one order and the +// stock items of that order's lines, which PlaceOrder bounded, and the adapter bounds the order ID. +export const cancelOrderDeclaration: CommandDeclaration< + Infer, + Infer +> = { + name: "CancelOrder", + contractVersion: 1, + input: cancelOrderInput, + output: cancelOrderResult, + permission: { permission: cancelOrderPermission }, + writes: [orderSource, stockItemSource], + readModels: [{ readModel: orderSummary, source: orderSource }], + rejections: [ + "orderNotFound", + "orderAlreadyCancelled", + "insufficientAllocation", + ], + executor: async (ctx, { tenantId, actor, operation, input }) => { + const { orderId } = input; + const cancelled = await ctx.runMutation( + components.orders.operations.cancel, + { tenantId, actor, operation, input: { orders: [{ orderId }] } }, + ); + const [order] = cancelled.result.orders; + if (order === undefined || cancelled.result.orders.length !== 1) + throw new Error( + `Orders answered ${cancelled.result.orders.length} orders for one`, + ); + // The quantities released are the cancelled order's own lines, as Orders returned them in this + // mutation: the stock item keeps totals only and remembers no order. + const released = await ctx.runMutation( + components.inventory.operations.release, + { + tenantId, + actor, + operation, + input: { + orderId, + lines: order.lines.map(({ stockItemId, quantity }) => ({ + stockItemId, + quantity, + })), + }, + }, + ); + // Neither context records a business failure, and CancelOrder declares none. + if (cancelled.kind !== "applied" || released.kind !== "applied") + throw new Error( + `CancelOrder received a ${cancelled.kind} from Orders and a ${released.kind} from Inventory`, + ); + return { + kind: "applied", + result: { orderId, released: released.result.lines }, + versions: [...cancelled.versions, ...released.versions], + streams: [...cancelled.streams, ...released.streams], + }; + }, +}; +export const cancelOrder = publicCommand(cancelOrderDeclaration); +export const cancelOrderInternal = internalCommand(cancelOrderDeclaration); diff --git a/example/convex/orders/_generated/component.ts b/example/convex/orders/_generated/component.ts index 719d9da..5e07c86 100644 --- a/example/convex/orders/_generated/component.ts +++ b/example/convex/orders/_generated/component.ts @@ -24,6 +24,105 @@ import type { FunctionReference } from "convex/server"; export type ComponentApi = { operations: { + cancel: FunctionReference< + "mutation", + "internal", + { + actor: { + delegationRef?: string; + id: string; + issuer?: string; + kind: "human" | "service" | "agent" | "reviewer" | "operator"; + onBehalfOf?: { + id: string; + kind: "human" | "service" | "agent" | "reviewer" | "operator"; + }; + }; + facts?: Record; + input: { orders: Array<{ orderId: string }> }; + operation: { + causedBy: + | { commandType: string; kind: "command" } + | { + contextId: string; + eventId: string; + kind: "event"; + tenantId: string; + } + | { kind: "migration"; migrationName: string }; + correlationId?: string; + operationId: string; + }; + tenantId: string; + }, + { + kind: "applied" | "businessFailure"; + result: { + orders: Array<{ + lines: Array<{ + quantity: number; + stockItemId: string; + unitPrice: number; + }>; + orderId: string; + }>; + }; + streams: Array<{ + appended: number; + created: boolean; + dto: any; + events: Array<{ + actor: { + delegationRef?: string; + id: string; + issuer?: string; + kind: "human" | "service" | "agent" | "reviewer" | "operator"; + onBehalfOf?: { + id: string; + kind: "human" | "service" | "agent" | "reviewer" | "operator"; + }; + }; + causedBy: + | { commandType: string; kind: "command" } + | { + contextId: string; + eventId: string; + kind: "event"; + tenantId: string; + } + | { kind: "migration"; migrationName: string }; + contextId: string; + correlationId?: string; + eventId: string; + eventSchemaVersion: number; + eventType: string; + occurredAt?: number; + operationId: string; + payload: any; + recordedAt: number; + streamId: string; + streamType: string; + streamVersion: number; + tenantId: string; + }>; + version: { + contextId: string; + streamId: string; + streamType: string; + tenantId: string; + version: number; + }; + }>; + versions: Array<{ + contextId: string; + streamId: string; + streamType: string; + tenantId: string; + version: number; + }>; + }, + Name + >; place: FunctionReference< "mutation", "internal", @@ -144,7 +243,7 @@ export type ComponentApi = }>; orderId: string; placedAt: number; - status: "placed"; + status: "placed" | "cancelled"; total: number; version: { contextId: string; @@ -182,7 +281,7 @@ export type ComponentApi = }>; orderId: string; placedAt: number; - status: "placed"; + status: "placed" | "cancelled"; total: number; version: { contextId: string; diff --git a/example/convex/orders/operations.ts b/example/convex/orders/operations.ts index 7e52313..68d9014 100644 --- a/example/convex/orders/operations.ts +++ b/example/convex/orders/operations.ts @@ -1,4 +1,4 @@ -// The Orders context's sanctioned operation. It takes a list, so a parent use case makes one call. +// The Orders context's sanctioned operations. Each takes a list, so a parent use case makes one call. import { v, type Infer, type ObjectType } from "convex/values"; import { defineOperation, @@ -6,7 +6,12 @@ import { type StreamResult, } from "../../../src/context/index.js"; import type { OrderResult } from "../../domain/index.js"; -import { journal, orderLineValidator, orderStream } from "./streams.js"; +import { + journal, + orderLineValidator, + orderStream, + type OrderDto, +} from "./streams.js"; const placeInput = { orders: v.array( v.object({ orderId: v.string(), lines: v.array(orderLineValidator) }), @@ -39,3 +44,32 @@ export const place = defineOperation< }), maxStreams: 32, }); +const cancelInput = { orders: v.array(v.object({ orderId: v.string() })) }; +const cancelResult = v.object({ + orders: v.array( + v.object({ orderId: v.string(), lines: v.array(orderLineValidator) }), + ), +}); +// Each order is cancelled with no expected version, so its own state decides: an order with no event +// is refused orderNotFound and a cancelled one orderAlreadyCancelled. Each cancelled order's lines +// come from its DTO, for the use case to release. +export const cancel = defineOperation< + ObjectType, + Infer +>(journal, { + name: "cancel", + streams: [orderStream], + input: cancelInput, + returns: cancelResult, + plan: ({ orders }) => + orders.map(({ orderId }) => + planned(orderStream, orderId, { commandType: "cancel" }), + ), + combine: (results) => ({ + orders: results.map((result) => ({ + orderId: result.version.streamId, + lines: (result.dto as OrderDto).lines, + })), + }), + maxStreams: 32, +}); diff --git a/example/convex/orders/streams.ts b/example/convex/orders/streams.ts index 04a53ed..9bcb476 100644 --- a/example/convex/orders/streams.ts +++ b/example/convex/orders/streams.ts @@ -24,7 +24,7 @@ export const orderLineValidator = v.object({ }); export const orderDtoValidator = v.object({ orderId: v.string(), - status: v.literal("placed"), + status: v.union(v.literal("placed"), v.literal("cancelled")), lines: v.array(orderLineValidator), total: v.number(), placedAt: v.number(), @@ -46,6 +46,7 @@ export const orderStream: StreamRegistration< lines: v.array(orderLineValidator), total: v.number(), }), + OrderCancelled: v.object({}), }, dto: orderDtoValidator, // A stream row exists only once its order is placed, so an order with no event has no DTO. diff --git a/example/domain/order.ts b/example/domain/order.ts index dff5600..d1763e5 100644 --- a/example/domain/order.ts +++ b/example/domain/order.ts @@ -1,5 +1,5 @@ -// The Orders context's order stream: an order is placed once with its lines, and its total is the sum -// of quantity times unit price over its lines. +// The Orders context's order stream: an order is placed once with its lines, its total is the sum of +// quantity times unit price over its lines, and a placed order is cancelled once. import type { DecideResult, Decider, @@ -14,19 +14,20 @@ export type OrderLine = { unitPrice: number; }; // none is a stream with no events yet, the status initial() returns. placedAt is the time the -// OrderPlaced event records, null before it. -export type OrderStatus = "none" | "placed"; +// OrderPlaced event records, null before it. A cancelled order keeps its lines, total and placedAt. +export type OrderStatus = "none" | "placed" | "cancelled"; export type OrderState = { status: OrderStatus; lines: OrderLine[]; total: number; placedAt: number | null; }; -export type OrderCommand = { commandType: "place"; lines: OrderLine[] }; -export type OrderEvent = DomainEvent< - "OrderPlaced", - { lines: OrderLine[]; total: number } ->; +export type OrderCommand = + { commandType: "place"; lines: OrderLine[] } | { commandType: "cancel" }; +export type OrderEvent = + | DomainEvent<"OrderPlaced", { lines: OrderLine[]; total: number }> + | DomainEvent<"OrderCancelled", Record>; +// The line count and total of the order placed or cancelled. export type OrderResult = { lineCount: number; total: number }; export const orderRejectionCodes = { // place with a quantity that is not a positive whole number, a unit price that is not a whole number @@ -34,6 +35,10 @@ export const orderRejectionCodes = { invalidQuantity: "invalidQuantity", // place on an order that is already placed. orderAlreadyPlaced: "orderAlreadyPlaced", + // cancel of an order with no event. + orderNotFound: "orderNotFound", + // cancel of an order that is already cancelled. + orderAlreadyCancelled: "orderAlreadyCancelled", } as const; const isQuantity = (quantity: number) => Number.isSafeInteger(quantity) && quantity >= 1; @@ -46,11 +51,41 @@ const eventSchemaVersion = 1; function reject(rejection: Rejection): DecideResult { return { kind: "rejection", rejection }; } +// A placed order records OrderCancelled; the order's own state refuses an order that has no event and +// one already cancelled, so a second cancel never reaches another context. +function cancel( + state: OrderState, + context: DecisionContext, +): DecideResult { + if (state.status === "none") + return reject({ + code: orderRejectionCodes.orderNotFound, + message: "The order does not exist", + }); + if (state.status === "cancelled") + return reject({ + code: orderRejectionCodes.orderAlreadyCancelled, + message: "The order is already cancelled", + }); + return { + kind: "applied", + events: [ + { + eventType: "OrderCancelled", + eventSchemaVersion, + payload: {}, + occurredAt: context.now, + }, + ], + result: { lineCount: state.lines.length, total: state.total }, + }; +} function decide( state: OrderState, command: OrderCommand, context: DecisionContext, ): DecideResult { + if (command.commandType === "cancel") return cancel(state, context); if (state.status !== "none") return reject({ code: orderRejectionCodes.orderAlreadyPlaced, @@ -97,6 +132,8 @@ function evolve(state: OrderState, event: OrderEvent): OrderState { total: event.payload.total, placedAt: event.occurredAt ?? null, }; + case "OrderCancelled": + return { ...state, status: "cancelled" }; } } export const orderDecider: Decider< @@ -111,7 +148,8 @@ export const orderDecider: Decider< evolve, invariants: [ { - // An order with no line is refused before it reaches the decider, as invalid input. + // An order with no line is refused before it reaches the decider, as invalid input. A cancelled + // order was placed and keeps what it was placed with. name: "aPlacedOrderHasALineATimeAndItsTotal", holds: (state) => state.status === "none" || diff --git a/example/domain/stockItem.ts b/example/domain/stockItem.ts index d084965..699387d 100644 --- a/example/domain/stockItem.ts +++ b/example/domain/stockItem.ts @@ -1,5 +1,6 @@ // The Inventory context's stock item stream: the quantity on hand and the quantity allocated to -// orders. The quantity available is the first less the second, and an allocation never passes it. +// orders, two totals and nothing per order. The quantity available is the first less the second, an +// allocation never passes it, and a release never passes the quantity allocated. import type { DecideResult, Decider, @@ -10,16 +11,20 @@ import type { export type StockItemState = { onHand: number; allocated: number }; export type StockItemCommand = | { commandType: "receive"; quantity: number } - | { commandType: "allocate"; orderId: string; quantity: number }; + | { commandType: "allocate"; orderId: string; quantity: number } + | { commandType: "release"; orderId: string; quantity: number }; export type StockItemEvent = | DomainEvent<"StockReceived", { quantity: number }> - | DomainEvent<"StockAllocated", { orderId: string; quantity: number }>; -// The quantity the command received or allocated. + | DomainEvent<"StockAllocated", { orderId: string; quantity: number }> + | DomainEvent<"AllocationReleased", { orderId: string; quantity: number }>; +// The quantity the command received, allocated or released. export type StockItemResult = { quantity: number }; export const stockItemRejectionCodes = { // allocate of more than the quantity available. insufficientStock: "insufficientStock", - // receive or allocate with a quantity that is not a positive whole number. + // release of more than the quantity allocated. + insufficientAllocation: "insufficientAllocation", + // receive, allocate or release with a quantity that is not a positive whole number. invalidQuantity: "invalidQuantity", // receive that would take the quantity on hand past the largest safe whole number. stockLimitExceeded: "stockLimitExceeded", @@ -35,7 +40,8 @@ function reject( ): DecideResult { return { kind: "rejection", rejection }; } -// A stock item with no stream decides against initial(), so an allocation of one is short of stock. +// A stock item with no stream decides against initial(), so an allocation of one is short of stock and +// a release of one is above the quantity allocated. function decide( state: StockItemState, command: StockItemCommand, @@ -68,6 +74,26 @@ function decide( result: { quantity }, }; } + if (command.commandType === "release") { + if (quantity > state.allocated) + return reject({ + code: stockItemRejectionCodes.insufficientAllocation, + message: `Cannot release ${quantity} when ${state.allocated} are allocated`, + details: { requested: quantity, allocated: state.allocated }, + }); + return { + kind: "applied", + events: [ + { + eventType: "AllocationReleased", + eventSchemaVersion, + payload: { orderId: command.orderId, quantity }, + occurredAt: context.now, + }, + ], + result: { quantity }, + }; + } if (!Number.isSafeInteger(state.onHand + quantity)) return reject({ code: stockItemRejectionCodes.stockLimitExceeded, @@ -93,6 +119,8 @@ function evolve(state: StockItemState, event: StockItemEvent): StockItemState { return { ...state, onHand: state.onHand + event.payload.quantity }; case "StockAllocated": return { ...state, allocated: state.allocated + event.payload.quantity }; + case "AllocationReleased": + return { ...state, allocated: state.allocated - event.payload.quantity }; } } export const stockItemDecider: Decider< diff --git a/tests/native/cancel-order-steps.ts b/tests/native/cancel-order-steps.ts new file mode 100644 index 0000000..83f29a0 --- /dev/null +++ b/tests/native/cancel-order-steps.ts @@ -0,0 +1,277 @@ +// The steps of spec:application.orders-inventory-example's example space, shared by its native +// examples. They run CancelOrder on the production composition: admin access seeds the grants, runs +// the order summary's first activation and, for one example, calls the Inventory context's release +// operation as setup, and reads stored documents and the function log; every command and query is sent +// by an ordinary client whose token the fixture issuer signs. +import type { ConvexHttpClient } from "convex/browser"; +import { getFunctionName, type FunctionReturnType } from "convex/server"; +import { ConvexError, type Value } from "convex/values"; +import { expect } from "vitest"; +import { api } from "../../example/convex/_generated/api.js"; +import { cancelOrderPermission } from "../../example/convex/ordering.js"; +import type { CompletionRecord, LogMark } from "../../harness/admin.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { required } from "../../harness/native.js"; +import { + changedDocuments, + grant, + line, + orderWorld, + permissions, + receiveStock, + stored, + tenantId, + type OrderWorld, + type Stored, +} from "./order-use-case.js"; +export const stockItemId = "sku-1"; +export const orderId = "order-1"; +// The order ID a cancel of an order never placed names. +export const neverPlaced = "order-2"; +// A second user of the fixture issuer, who holds every grant of the composition except orders.cancel. +const lackingUser = "user-2"; +export const cancelOrderIdentifier = getFunctionName(api.ordering.cancelOrder); +type CancelResponse = FunctionReturnType; +type PlaceResponse = FunctionReturnType; +export interface CancelWorld { + order?: OrderWorld; + ordered?: number; + // The stock item's stream row as it was before PlaceOrder allocated. + stockBefore?: Record; + placed?: PlaceResponse; + // The cancel a Given step sent under k-1. + first?: CancelResponse; + target?: string; + key?: string; + before?: Stored; + mark?: LogMark; + response?: CancelResponse; + error?: unknown; +} +const backendOf = (world: CancelWorld) => + required(world.order, "the backend").backend; +async function stockRow(world: CancelWorld) { + const rows = await backendOf(world).admin.readTable("streams", { + component: "inventory", + }); + return required( + rows.find((row) => row["streamId"] === stockItemId), + "the stock item's stream row", + ); +} +export const stockState = async (world: CancelWorld) => + (await stockRow(world))["state"] as { onHand: number; allocated: number }; +export async function unitsReceived(world: CancelWorld, onHand: number) { + world.order = await orderWorld(); + await receiveStock(world.order, [{ stockItemId, quantity: onHand }]); +} +// Two lines on the one stock item, of ordered less one and of one unit, which Inventory sums. +export async function orderPlaced(world: CancelWorld, ordered: number) { + const order = required(world.order, "the backend"); + if (ordered < 2) throw new Error("Two lines need at least two units"); + world.stockBefore = await stockRow(world); + world.placed = await order.client.mutation(api.ordering.placeOrder, { + tenantId, + requestKey: "k-place", + input: { + orderId, + lines: [line(stockItemId, ordered - 1, 250), line(stockItemId, 1, 999)], + }, + }); + expect(world.placed).toMatchObject({ kind: "applied", replayed: false }); + world.ordered = ordered; +} +export async function orderBefore( + world: CancelWorld, + before: + | "is cancelled under request key k-1" + | "has its units released by Inventory alone", +) { + const order = required(world.order, "the backend"); + if (before === "is cancelled under request key k-1") { + world.first = await order.client.mutation(api.ordering.cancelOrder, { + tenantId, + requestKey: "k-1", + input: { orderId }, + }); + expect(world.first).toMatchObject({ kind: "applied", replayed: false }); + return; + } + // No command releases stock without cancelling its order, so the setup calls the Inventory + // context's own operation with admin access; its state stays the fold of its events. + const outcome = await order.backend.admin.run( + "operations:release", + { + tenantId, + actor: { kind: "operator", id: "native-test" }, + operation: { + operationId: "setup-release", + causedBy: { kind: "command", commandType: "Setup" }, + }, + input: { + orderId, + lines: [ + { stockItemId, quantity: required(world.ordered, "the order") }, + ], + }, + }, + { component: "inventory" }, + ); + expect(outcome).toMatchObject({ kind: "applied" }); +} +async function lackingClient(order: OrderWorld): Promise { + for (const permission of permissions) + if (permission !== cancelOrderPermission) + await grant(order.backend, lackingUser, permission); + return ordinaryClient(order.backend.url, { + token: await order.backend.issuer.token(lackingUser), + }); +} +export async function cancelSent( + world: CancelWorld, + { + grant: holds, + target, + key, + }: { + grant: "holding" | "lacking"; + target: "that order" | "an order never placed"; + key: string; + }, +) { + const order = required(world.order, "the backend"); + const client = + holds === "holding" ? order.client : await lackingClient(order); + world.target = target === "that order" ? orderId : neverPlaced; + world.key = key; + world.before = await stored(order); + world.mark = await order.backend.admin.logMark(); + try { + world.response = await client.mutation(api.ordering.cancelOrder, { + tenantId, + requestKey: key, + input: { orderId: world.target }, + }); + } catch (error) { + world.error = error; + } +} +export function callerReceives( + world: CancelWorld, + response: "the result" | "the receipt's answer" | "the rejection", +) { + if (response === "the rejection") { + expect(world.response).toBeUndefined(); + expect(world.error).toBeInstanceOf(ConvexError); + expect(rejectionData(world)).toMatchObject({ + kind: "rejection", + commandType: "CancelOrder", + }); + return; + } + expect(world.error).toBeUndefined(); + if (response === "the result") { + expect(world.response).toMatchObject({ kind: "applied", replayed: false }); + return; + } + const first = required(world.first, "the first cancel"); + expect(world.response).toEqual({ + kind: "applied", + result: null, + operationId: first.operationId, + affected: first.affected, + versions: first.versions, + replayed: true, + }); +} +export const rejectionData = (world: CancelWorld) => + (world.error as ConvexError>).data; +export function rejectionCodeIs(world: CancelWorld, code: string) { + expect(rejectionData(world)["code"]).toBe(code); +} +export async function allocatedIs(world: CancelWorld, allocated: number) { + expect((await stockState(world)).allocated).toBe(allocated); +} +// The table holds the one summary row of the order placed, whatever order the cancel named. +export async function summaryStatusIs( + world: CancelWorld, + status: "placed" | "cancelled", +) { + const rows = await backendOf(world).admin.readTable("orderSummaries"); + expect(rows).toMatchObject([{ key: orderId, status }]); +} +export async function writtenIs(world: CancelWorld, written: number) { + const after = await stored(required(world.order, "the backend")); + expect( + changedDocuments(required(world.before, "the stored documents"), after), + ).toBe(written); +} +// The function log's records of the one request that ran CancelOrder after the mark. +export async function cancelRequest(world: CancelWorld): Promise<{ + record: CompletionRecord; + request: CompletionRecord[]; +}> { + const own = (entry: { identifier: string; componentPath: string | null }) => + entry.identifier === cancelOrderIdentifier && entry.componentPath === null; + const records = await backendOf(world).admin.completionsSince( + required(world.mark, "the log mark"), + (entries) => entries.some(own), + ); + const record = required(records.find(own), "the command's completion record"); + return { + record, + request: records.filter((entry) => entry.requestId === record.requestId), + }; +} +// Every step of the example space, for an example to bind the ones its fence names. +export const steps = { + "{onHand} units of a stock item received through ReceiveStock": ( + world: CancelWorld, + { onHand }: { onHand: number }, + ) => unitsReceived(world, onHand), + "an order placed for {ordered} of them over two lines": ( + world: CancelWorld, + { ordered }: { ordered: number }, + ) => orderPlaced(world, ordered), + "the order {before}": ( + world: CancelWorld, + { + before, + }: { + before: + | "is cancelled under request key k-1" + | "has its units released by Inventory alone"; + }, + ) => orderBefore(world, before), + "a caller {grant} the permission orders.cancel sends CancelOrder for {target} under request key {key}": + ( + world: CancelWorld, + params: { + grant: "holding" | "lacking"; + target: "that order" | "an order never placed"; + key: string; + }, + ) => cancelSent(world, params), + "the caller receives {response}": ( + world: CancelWorld, + { + response, + }: { response: "the result" | "the receipt's answer" | "the rejection" }, + ) => callerReceives(world, response), + "the rejection code is {code}": ( + world: CancelWorld, + { code }: { code: string }, + ) => rejectionCodeIs(world, code), + "the stock item's quantity allocated is {allocated}": ( + world: CancelWorld, + { allocated }: { allocated: number }, + ) => allocatedIs(world, allocated), + "the order summary's status is {status}": ( + world: CancelWorld, + { status }: { status: "placed" | "cancelled" }, + ) => summaryStatusIs(world, status), + "the number of documents the command wrote is {written}": ( + world: CancelWorld, + { written }: { written: number }, + ) => writtenIs(world, written), +}; diff --git a/tests/native/order-use-case.ts b/tests/native/order-use-case.ts index f18f790..4675a98 100644 --- a/tests/native/order-use-case.ts +++ b/tests/native/order-use-case.ts @@ -1,4 +1,4 @@ -// Shared steps of the scenarios that run PlaceOrder on the production composition: a backend whose +// Shared steps of the scenarios that run PlaceOrder and CancelOrder on the production composition: a backend whose // user holds every grant the use cases need, the order summary's first activation, stock through // ReceiveStock, and every stored document of both contexts and the parent, to count what a call // changed. Admin access is used for setup and for reading stored documents only. @@ -6,7 +6,10 @@ import { getFunctionName } from "convex/server"; import type { ConvexHttpClient } from "convex/browser"; import type { Value } from "convex/values"; import { api, internal } from "../../example/convex/_generated/api.js"; -import { placeOrderPermission } from "../../example/convex/ordering.js"; +import { + cancelOrderPermission, + placeOrderPermission, +} from "../../example/convex/ordering.js"; import { readOrdersPermission } from "../../example/convex/readModels.js"; import { receiveStockPermission } from "../../example/convex/receiving.js"; import type { Backend } from "../../harness/backend.js"; @@ -14,8 +17,9 @@ import { ordinaryClient } from "../../harness/clients.js"; import { productionBackend } from "../../harness/native.js"; export const tenantId = "t-1"; const subject = "user-1"; -const permissions = [ +export const permissions = [ placeOrderPermission, + cancelOrderPermission, receiveStockPermission, readOrdersPermission, "inventory.read", diff --git a/tests/native/orders-inventory-example.cancel-duplicate-answered-from-receipt.test.ts b/tests/native/orders-inventory-example.cancel-duplicate-answered-from-receipt.test.ts new file mode 100644 index 0000000..193ce55 --- /dev/null +++ b/tests/native/orders-inventory-example.cancel-duplicate-answered-from-receipt.test.ts @@ -0,0 +1,40 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { cancelDuplicateAnsweredFromReceiptContract as contract } from "../../generated/contracts/application.orders-inventory-example.cancel-duplicate-answered-from-receipt.contract.js"; +import { required } from "../../harness/native.js"; +import { steps, type CancelWorld } from "./cancel-order-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.orders-inventory-example.cancel-duplicate-answered-from-receipt", + ), + verifies: ref( + "spec:application.orders-inventory-example.cancel-duplicate-answered-from-receipt", + ), +}); +void anchor; +// The cancel of order-1 is sent again under the same key k-1 and is answered from its receipt. +bindExample( + contract, + (): CancelWorld => ({}), + steps, + async (world) => { + const { backend } = required(world.order, "the backend"); + const first = required(world.first, "the first cancel"); + expect(world.response).toMatchObject({ + kind: "applied", + replayed: true, + result: null, + operationId: first.operationId, + }); + expect( + (await backend.admin.readTable("receipts")).filter( + (row) => row["requestKey"] === "k-1", + ), + ).toMatchObject([{ operationId: first.operationId }]); + }, +); diff --git a/tests/native/orders-inventory-example.cancel-placed-order.test.ts b/tests/native/orders-inventory-example.cancel-placed-order.test.ts new file mode 100644 index 0000000..98ba523 --- /dev/null +++ b/tests/native/orders-inventory-example.cancel-placed-order.test.ts @@ -0,0 +1,123 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { cancelPlacedOrderContract as contract } from "../../generated/contracts/application.orders-inventory-example.cancel-placed-order.contract.js"; +import { api } from "../../example/convex/_generated/api.js"; +import { required } from "../../harness/native.js"; +import { + cancelRequest, + orderId, + stockItemId, + stockState, + steps, + type CancelWorld, +} from "./cancel-order-steps.js"; +import { tenantId } from "./order-use-case.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.orders-inventory-example.cancel-placed-order", + ), + verifies: ref( + "spec:application.orders-inventory-example.cancel-placed-order", + ), +}); +void anchor; +// CancelOrder on the production composition cancels order-1 in Orders and then releases its two +// lines, summed, on sku-1 in Inventory: one mutation, one receipt, and the summary row replaced. +bindExample( + contract, + (): CancelWorld => ({}), + steps, + async (world) => { + const order = required(world.order, "the backend"); + const response = required(world.response, "the response"); + // The stock item's totals are what they were before PlaceOrder allocated. + const before = required(world.stockBefore, "the stock before the order"); + expect(await stockState(world)).toEqual(before["state"]); + expect(await stockState(world)).toEqual({ onHand: 5, allocated: 0 }); + const orderVersion = { + tenantId, + contextId: "orders", + streamType: "order", + streamId: orderId, + version: 2, + }; + expect(response).toMatchObject({ + kind: "applied", + replayed: false, + result: { orderId, released: [{ stockItemId, quantity: 3 }] }, + versions: [ + orderVersion, + { + tenantId, + contextId: "inventory", + streamType: "stockItem", + streamId: stockItemId, + version: 3, + }, + ], + }); + const ofOperation = async (component: string) => + (await order.backend.admin.readTable("events", { component })).filter( + (row) => row["operationId"] === response.operationId, + ); + expect(await ofOperation("orders")).toMatchObject([ + { eventType: "OrderCancelled", streamId: orderId, payload: {} }, + ]); + expect(await ofOperation("inventory")).toMatchObject([ + { + eventType: "AllocationReleased", + streamId: stockItemId, + payload: { orderId, quantity: 3 }, + }, + ]); + expect( + (await order.backend.admin.readTable("receipts")).filter( + (row) => row["requestKey"] === "k-1", + ), + ).toMatchObject([{ operationId: response.operationId }]); + expect(await order.backend.admin.readTable("orderSummaries")).toMatchObject( + [ + { + key: orderId, + sourceVersions: [orderVersion], + lineCount: 2, + total: 1499, + }, + ], + ); + // The command ran as one top-level mutation, which committed. + const { record, request } = await cancelRequest(world); + expect(request).toHaveLength(1); + expect(record.error).toBeNull(); + // The parent reads the cancelled order with what it was placed with, and lists it by its status. + const placed = await order.client.query(api.orderQueries.getOrder, { + tenantId, + orderId, + }); + expect(placed).toMatchObject({ + orderId, + status: "cancelled", + lines: [ + { stockItemId, quantity: 2, unitPrice: 250 }, + { stockItemId, quantity: 1, unitPrice: 999 }, + ], + total: 1499, + version: orderVersion, + }); + const list = (status: "placed" | "cancelled") => + order.client.query(api.readModels.listOrderSummaries, { + tenantId, + status, + paginationOpts: { cursor: null, numItems: 10 }, + }); + expect((await list("cancelled")).page).toMatchObject([ + { orderId, status: "cancelled", placedAt: placed?.placedAt }, + ]); + expect((await list("placed")).page).toEqual([]); + }, +); diff --git a/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts b/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts new file mode 100644 index 0000000..e1c649c --- /dev/null +++ b/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts @@ -0,0 +1,61 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { cancelSecondContextRejectsContract as contract } from "../../generated/contracts/application.orders-inventory-example.cancel-second-context-rejects.contract.js"; +import { required } from "../../harness/native.js"; +import { + cancelRequest, + orderId, + rejectionData, + steps, + type CancelWorld, +} from "./cancel-order-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.orders-inventory-example.cancel-second-context-rejects", + ), + verifies: ref( + "spec:application.orders-inventory-example.cancel-second-context-rejects", + ), +}); +void anchor; +// The setup released order-1's units in Inventory alone, so CancelOrder's Orders call records the +// cancel and returns the order's lines, and Inventory then refuses their release. The refusal is +// the kernel's bare Rejection thrown in Inventory's sub-transaction, and the whole mutation throws it. +bindExample( + contract, + (): CancelWorld => ({}), + steps, + async (world) => { + // Only a release planned from the lines the Orders call returned asks for 3. + expect(rejectionData(world)).toEqual({ + kind: "rejection", + code: "insufficientAllocation", + commandType: "CancelOrder", + message: "Cannot release 3 when 0 are allocated", + details: { requested: 3, allocated: 0 }, + }); + const { record, request } = await cancelRequest(world); + expect(request).toHaveLength(1); + expect(record.error).toMatch(/^Uncaught ConvexError: /); + expect(record.usageStats["databaseWriteDocuments"]).toBe(0); + const { backend } = required(world.order, "the backend"); + const own = (rows: Record[]) => + rows.filter((row) => row["streamId"] === orderId); + expect( + own(await backend.admin.readTable("streams", { component: "orders" })), + ).toMatchObject([{ streamVersion: 1, state: { status: "placed" } }]); + expect( + own(await backend.admin.readTable("events", { component: "orders" })), + ).toMatchObject([{ eventType: "OrderPlaced" }]); + expect( + (await backend.admin.readTable("receipts")).filter( + (row) => row["requestKey"] === "k-1", + ), + ).toEqual([]); + }, +); diff --git a/tests/native/orders-inventory-example.cancel-twice-rejected.test.ts b/tests/native/orders-inventory-example.cancel-twice-rejected.test.ts new file mode 100644 index 0000000..25d0af6 --- /dev/null +++ b/tests/native/orders-inventory-example.cancel-twice-rejected.test.ts @@ -0,0 +1,46 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { cancelTwiceRejectedContract as contract } from "../../generated/contracts/application.orders-inventory-example.cancel-twice-rejected.contract.js"; +import { required } from "../../harness/native.js"; +import { + rejectionData, + stockState, + steps, + type CancelWorld, +} from "./cancel-order-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.orders-inventory-example.cancel-twice-rejected", + ), + verifies: ref( + "spec:application.orders-inventory-example.cancel-twice-rejected", + ), +}); +void anchor; +// A second CancelOrder of order-1 under a new key is refused by the order's own state in Orders. +bindExample( + contract, + (): CancelWorld => ({}), + steps, + async (world) => { + expect(rejectionData(world)).toEqual({ + kind: "rejection", + code: "orderAlreadyCancelled", + commandType: "CancelOrder", + message: "The order is already cancelled", + }); + const { backend } = required(world.order, "the backend"); + expect( + (await backend.admin.readTable("receipts")).filter( + (row) => row["requestKey"] === "k-2", + ), + ).toEqual([]); + // Had Inventory decided, its release of 3 would have been refused insufficientAllocation. + expect(await stockState(world)).toEqual({ onHand: 5, allocated: 0 }); + }, +); diff --git a/tests/native/orders-inventory-example.cancel-unknown-order-rejected.test.ts b/tests/native/orders-inventory-example.cancel-unknown-order-rejected.test.ts new file mode 100644 index 0000000..383c89b --- /dev/null +++ b/tests/native/orders-inventory-example.cancel-unknown-order-rejected.test.ts @@ -0,0 +1,54 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { cancelUnknownOrderRejectedContract as contract } from "../../generated/contracts/application.orders-inventory-example.cancel-unknown-order-rejected.contract.js"; +import { api } from "../../example/convex/_generated/api.js"; +import { required } from "../../harness/native.js"; +import { + neverPlaced, + rejectionData, + steps, + type CancelWorld, +} from "./cancel-order-steps.js"; +import { tenantId } from "./order-use-case.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.orders-inventory-example.cancel-unknown-order-rejected", + ), + verifies: ref( + "spec:application.orders-inventory-example.cancel-unknown-order-rejected", + ), +}); +void anchor; +// CancelOrder for order-2, which no PlaceOrder recorded: the order decider decides it against the +// initial state and refuses it, and the adapter creates no stream for it. +bindExample( + contract, + (): CancelWorld => ({}), + steps, + async (world) => { + expect(rejectionData(world)).toEqual({ + kind: "rejection", + code: "orderNotFound", + commandType: "CancelOrder", + message: "The order does not exist", + }); + const order = required(world.order, "the backend"); + for (const table of ["streams", "events"]) + expect( + ( + await order.backend.admin.readTable(table, { component: "orders" }) + ).filter((row) => row["streamId"] === neverPlaced), + ).toEqual([]); + expect( + await order.client.query(api.orderQueries.getOrder, { + tenantId, + orderId: neverPlaced, + }), + ).toBeNull(); + }, +); diff --git a/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts b/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts new file mode 100644 index 0000000..87061bf --- /dev/null +++ b/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts @@ -0,0 +1,37 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { cancelWithoutGrantRefusedContract as contract } from "../../generated/contracts/application.orders-inventory-example.cancel-without-grant-refused.contract.js"; +import { + rejectionData, + steps, + type CancelWorld, +} from "./cancel-order-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.orders-inventory-example.cancel-without-grant-refused", + ), + verifies: ref( + "spec:application.orders-inventory-example.cancel-without-grant-refused", + ), +}); +void anchor; +// A second user who holds every other grant of the composition, but not orders.cancel, is refused. +bindExample( + contract, + (): CancelWorld => ({}), + steps, + (world) => { + expect(rejectionData(world)).toEqual({ + kind: "rejection", + code: "forbidden", + commandType: "CancelOrder", + message: "The caller may not run CancelOrder in this tenant", + details: { reason: "no_grant" }, + }); + }, +); diff --git a/tests/pure/example-domain.test.ts b/tests/pure/example-domain.test.ts index fc7b9ba..0d29597 100644 --- a/tests/pure/example-domain.test.ts +++ b/tests/pure/example-domain.test.ts @@ -7,6 +7,7 @@ import { expect, test } from "vitest"; import { checkInvariants, fold, + rebuild, type DecideResult, type DecisionContext, type DomainEvent, @@ -15,9 +16,11 @@ import { orderDecider, orderTotal, stockItemDecider, + type OrderEvent, type OrderLine, type OrderState, type StockItemCommand, + type StockItemEvent, type StockItemState, } from "../../example/domain/index.js"; import { orderStream } from "../../example/convex/orders/streams.js"; @@ -28,7 +31,7 @@ import type { StreamMeta } from "../../src/context/index.js"; const anchor = codeAnchor({ id: codeAnchorId("impl:application.orders-inventory-example"), label: - "the order and stock item deciders, the Orders and Inventory contexts, PlaceOrder, ReceiveStock and the order summary", + "the order and stock item deciders, the Orders and Inventory contexts, PlaceOrder, CancelOrder, ReceiveStock and the order summary", satisfies: ref("spec:application.orders-inventory-example"), }); void anchor; @@ -95,6 +98,81 @@ test("pure: an order is placed once", () => { }); }); +const cancelCommand = { commandType: "cancel" } as const; +const later: DecisionContext = { ...context, now: context.now + 60000 }; +const cancelled = (): OrderState => + fold( + orderDecider.evolve, + placed(), + applied(orderDecider.decide(placed(), cancelCommand, later)).events, + ); + +test("pure: a placed order is cancelled with one OrderCancelled event, and keeps its lines, total and time", () => { + expect(orderDecider.decide(placed(), cancelCommand, later)).toStrictEqual({ + kind: "applied", + events: [ + { + eventType: "OrderCancelled", + eventSchemaVersion: 1, + payload: {}, + occurredAt: later.now, + }, + ], + result: { lineCount: 3, total: 597 }, + }); + expect(cancelled()).toStrictEqual({ + status: "cancelled", + lines, + total: 597, + placedAt: context.now, + }); + expect(checkInvariants(orderDecider.invariants ?? [], cancelled())).toEqual( + [], + ); +}); + +test("pure: a cancel of an order with no event is refused orderNotFound, and a second cancel orderAlreadyCancelled", () => { + expect( + rejection( + orderDecider.decide(orderDecider.initial(), cancelCommand, context), + ), + ).toStrictEqual({ + code: "orderNotFound", + message: "The order does not exist", + }); + expect( + rejection(orderDecider.decide(cancelled(), cancelCommand, later)), + ).toStrictEqual({ + code: "orderAlreadyCancelled", + message: "The order is already cancelled", + }); + // A cancelled order is still one that was placed. + expect( + rejection( + orderDecider.decide(cancelled(), { commandType: "place", lines }, later), + ), + ).toMatchObject({ code: "orderAlreadyPlaced" }); +}); + +test("pure: an order rebuilt from OrderPlaced and OrderCancelled equals the state the two decisions folded", () => { + const events: OrderEvent[] = []; + let state = orderDecider.initial(); + for (const [command, at] of [ + [{ commandType: "place", lines }, context], + [cancelCommand, later], + ] as const) { + const decision = applied(orderDecider.decide(state, command, at)); + state = fold(orderDecider.evolve, state, decision.events); + events.push(...decision.events); + } + expect(events.map((event) => event.eventType)).toEqual([ + "OrderPlaced", + "OrderCancelled", + ]); + expect(rebuild(orderDecider, events)).toStrictEqual(state); + expect(state).toStrictEqual(cancelled()); +}); + test.each([ ["a quantity of zero", { quantity: 0, unitPrice: 1 }], ["a fractional quantity", { quantity: 1.5, unitPrice: 1 }], @@ -125,7 +203,7 @@ test("pure: an order whose total passes the largest safe whole number is refused ).toMatchObject({ code: "invalidQuantity", details: { total: big + 1 } }); }); -test("pure: the order invariant holds for no order and for a placed order, and fails for one with no line, no time or a wrong total", () => { +test("pure: the order invariant holds for no order, a placed order and a cancelled one, and fails for one with no line, no time or a wrong total", () => { const invariants = orderDecider.invariants ?? []; const name = "aPlacedOrderHasALineATimeAndItsTotal"; expect(checkInvariants(invariants, orderDecider.initial())).toEqual([]); @@ -138,6 +216,10 @@ test("pure: the order invariant holds for no order and for a placed order, and f expect(checkInvariants(invariants, { ...placed(), total: 596 })).toEqual([ name, ]); + expect(checkInvariants(invariants, cancelled())).toEqual([]); + expect( + checkInvariants(invariants, { ...cancelled(), lines: [], total: 0 }), + ).toEqual([name]); // An empty order reaches the invariant only if nothing before the decider refused it. const empty = applied(place([])); expect( @@ -175,6 +257,16 @@ test("pure: a placed order's DTO carries its lines, total, time and stream versi expect(() => orderStream.toDto(orderDecider.initial(), meta("order-2")), ).toThrow("Order order-2 is not placed and has no DTO"); + expect( + orderStream.toDto(cancelled(), { ...meta("order-1"), streamVersion: 2 }), + ).toMatchObject({ + orderId: "order-1", + status: "cancelled", + lines, + total: 597, + placedAt: context.now, + version: { streamId: "order-1", version: 2 }, + }); }); test("pure: the order summary projects the order's DTO to its status, line count, total and time, keyed by the order ID", () => { @@ -193,6 +285,14 @@ test("pure: the order summary projects the order's DTO to its status, line count }); // Deterministic: the same DTO gives the same row. expect(projection.project("t-1", dto, [dto.version])).toStrictEqual(row); + // A cancelled order's DTO gives the same row with the status cancelled. + const cancelledDto = orderStream.toDto(cancelled(), { + ...meta("order-1"), + streamVersion: 2, + }) as typeof dto; + expect( + projection.project("t-1", cancelledDto, [cancelledDto.version]), + ).toStrictEqual({ ...row, status: "cancelled" }); expect(orderSummary).toMatchObject({ name: "orderSummary", table: "orderSummaries", @@ -210,6 +310,11 @@ const allocate = (quantity: number): StockItemCommand => ({ orderId: "order-1", quantity, }); +const release = (quantity: number): StockItemCommand => ({ + commandType: "release", + orderId: "order-1", + quantity, +}); function stockAfter(commands: readonly StockItemCommand[]): StockItemState { let state = stockItemDecider.initial(); for (const command of commands) @@ -274,11 +379,80 @@ test("pure: an allocation above the quantity available is refused insufficientSt }); }); +test("pure: release subtracts from the quantity allocated with one AllocationReleased event and leaves the quantity on hand", () => { + expect( + stockItemDecider.decide({ onHand: 5, allocated: 3 }, release(3), context), + ).toStrictEqual({ + kind: "applied", + events: [ + { + eventType: "AllocationReleased", + eventSchemaVersion: 1, + payload: { orderId: "order-1", quantity: 3 }, + occurredAt: context.now, + }, + ], + result: { quantity: 3 }, + }); + // The totals are back to what they were before the allocation. + expect(stockAfter([receive(5), allocate(3), release(3)])).toEqual( + stockAfter([receive(5)]), + ); + expect( + stockAfter([receive(5), allocate(3), allocate(1), release(3)]), + ).toEqual({ onHand: 5, allocated: 1 }); +}); + +test("pure: a release above the quantity allocated is refused insufficientAllocation with the requested and the allocated quantity", () => { + const state = stockAfter([receive(5), allocate(2)]); + expect( + rejection(stockItemDecider.decide(state, release(3), context)), + ).toStrictEqual({ + code: "insufficientAllocation", + message: "Cannot release 3 when 2 are allocated", + details: { requested: 3, allocated: 2 }, + }); + // A release of exactly the quantity allocated is applied. + expect(stockItemDecider.decide(state, release(2), context).kind).toBe( + "applied", + ); + expect( + rejection( + stockItemDecider.decide(stockItemDecider.initial(), release(1), context), + ), + ).toMatchObject({ + code: "insufficientAllocation", + details: { requested: 1, allocated: 0 }, + }); +}); + +test("pure: a stock item rebuilt from its received, allocated and released events equals the state the decisions folded", () => { + const events: StockItemEvent[] = []; + let state = stockItemDecider.initial(); + for (const command of [receive(5), allocate(3), receive(2), release(3)]) { + const decision = applied(stockItemDecider.decide(state, command, context)); + state = fold(stockItemDecider.evolve, state, decision.events); + events.push(...decision.events); + } + expect(events.map((event) => event.eventType)).toEqual([ + "StockReceived", + "StockAllocated", + "StockReceived", + "AllocationReleased", + ]); + expect(state).toEqual({ onHand: 7, allocated: 0 }); + expect(rebuild(stockItemDecider, events)).toStrictEqual(state); +}); + test.each([0, -1, 1.5])( - "pure: receive and allocate refuse a quantity of %s as invalidQuantity", + "pure: receive, allocate and release refuse a quantity of %s as invalidQuantity", (quantity) => { - const state = stockAfter([receive(5)]); - for (const command of [receive(quantity), allocate(quantity)]) + const state = stockAfter([receive(5), allocate(5)]); + for (const command of [ + receive(quantity), + allocate(quantity), + release(quantity), + ]) expect( rejection(stockItemDecider.decide(state, command, context)), ).toEqual({ diff --git a/tests/pure/example-use-cases.test.ts b/tests/pure/example-use-cases.test.ts index 105b6b0..ad88207 100644 --- a/tests/pure/example-use-cases.test.ts +++ b/tests/pure/example-use-cases.test.ts @@ -5,15 +5,19 @@ import { } from "@libar-dev/software-delivery-protocol"; import { getFunctionAddress } from "convex/server"; import { expect, test } from "vitest"; -import { placeOrderDeclaration } from "../../example/convex/ordering.js"; +import { + cancelOrderDeclaration, + placeOrderDeclaration, +} from "../../example/convex/ordering.js"; import { receiveStockDeclaration } from "../../example/convex/receiving.js"; import type { MutationCtx } from "../../src/command/index.js"; import type { OperationRef, StreamDto } from "../../src/context/index.js"; import type { StreamVersion } from "../../src/kernel/index.js"; -// Binds PlaceOrder and ReceiveStock, the production composition's use cases, to their Spec. +// Binds PlaceOrder, CancelOrder and ReceiveStock, the production composition's use cases, to their Spec. const anchor = codeAnchor({ id: codeAnchorId("impl:application.parent-use-cases"), - label: "PlaceOrder over Orders and then Inventory, and ReceiveStock", + label: + "PlaceOrder and CancelOrder over Orders and then Inventory, and ReceiveStock", satisfies: ref("spec:application.parent-use-cases"), }); void anchor; @@ -54,6 +58,8 @@ function countingCtx(answers: Record) { const place = "_reference/childComponent/orders/operations/place"; const allocate = "_reference/childComponent/inventory/operations/allocate"; const receive = "_reference/childComponent/inventory/operations/receive"; +const cancel = "_reference/childComponent/orders/operations/cancel"; +const release = "_reference/childComponent/inventory/operations/release"; const actor = { kind: "human", id: "issuer|user-1" } as const; const operation: OperationRef = { operationId: "op-1", @@ -225,3 +231,128 @@ test("pure: PlaceOrder and ReceiveStock declare what they write, the order summa }); expect(receiveStockDeclaration.readModels).toBeUndefined(); }); + +// The lines Orders answers for the cancelled order: four lines over three stock items, with prices. +const cancelledLines = [0, 1, 2, 0].map((i) => ({ + stockItemId: `sku-${i}`, + quantity: 2 * i + 1, + unitPrice: 25, +})); +const released = [ + { stockItemId: "sku-0", quantity: 2 }, + { stockItemId: "sku-1", quantity: 3 }, + { stockItemId: "sku-2", quantity: 5 }, +]; +function cancelOrderAnswers( + kinds: { orders?: string; inventory?: string } = {}, +) { + return { + [cancel]: { + kind: kinds.orders ?? "applied", + result: { orders: [{ orderId: "order-1", lines: cancelledLines }] }, + versions: [orderVersion], + streams: [entry(orderVersion)], + }, + [release]: { + kind: kinds.inventory ?? "applied", + result: { lines: released }, + versions: stockVersions, + streams: stockVersions.map(entry), + }, + }; +} +const runCancel = (ctx: MutationCtx) => + cancelOrderDeclaration.executor(ctx, { + tenantId: "t-1", + actor, + operation, + input: { orderId: "order-1" }, + }); + +test("pure: CancelOrder's executor calls Orders' cancel once and then Inventory's release once with the lines Orders returned, and returns both calls' versions and streams in that order", async () => { + const { ctx, calls } = countingCtx(cancelOrderAnswers()); + const executed = await runCancel(ctx); + expect(calls.map((call) => call.reference)).toEqual([cancel, release]); + expect(calls[0]?.args).toEqual({ + tenantId: "t-1", + actor, + operation, + input: { orders: [{ orderId: "order-1" }] }, + }); + // Inventory is given the stock item and quantity of each line Orders returned, without its price: + // the quantities come from the cancelled order, not from the caller. + expect(calls[1]?.args).toEqual({ + tenantId: "t-1", + actor, + operation, + input: { + orderId: "order-1", + lines: cancelledLines.map(({ stockItemId, quantity }) => ({ + stockItemId, + quantity, + })), + }, + }); + expect(executed).toEqual({ + kind: "applied", + result: { orderId: "order-1", released }, + versions: [orderVersion, ...stockVersions], + streams: [entry(orderVersion), ...stockVersions.map(entry)], + }); +}); + +test("pure: CancelOrder's executor fails when either context answers a business failure, which CancelOrder does not declare", async () => { + for (const kinds of [ + { orders: "businessFailure" }, + { inventory: "businessFailure" }, + ]) + await expect( + runCancel(countingCtx(cancelOrderAnswers(kinds)).ctx), + ).rejects.toThrow( + `CancelOrder received a ${kinds.orders ?? "applied"} from Orders and a ${kinds.inventory ?? "applied"} from Inventory`, + ); +}); + +test("pure: CancelOrder's executor fails when Orders answers other than one order, before it calls Inventory", async () => { + for (const orders of [ + [], + [ + { orderId: "order-1", lines: cancelledLines }, + { orderId: "order-2", lines: cancelledLines }, + ], + ]) { + const answers = cancelOrderAnswers(); + (answers[cancel] as { result: { orders: unknown[] } }).result.orders = + orders; + const { ctx, calls } = countingCtx(answers); + await expect(runCancel(ctx)).rejects.toThrow( + `Orders answered ${orders.length} orders for one`, + ); + expect(calls.map((call) => call.reference)).toEqual([cancel]); + } +}); + +test("pure: CancelOrder declares its permission, what it writes, the order summary, its three codes and no bounds", () => { + expect(cancelOrderDeclaration).toMatchObject({ + name: "CancelOrder", + contractVersion: 1, + permission: { permission: "orders.cancel" }, + writes: [ + { contextId: "orders", streamType: "order" }, + { contextId: "inventory", streamType: "stockItem" }, + ], + readModels: [ + { + readModel: { name: "orderSummary" }, + source: { contextId: "orders", streamType: "order" }, + }, + ], + rejections: [ + "orderNotFound", + "orderAlreadyCancelled", + "insufficientAllocation", + ], + }); + expect(cancelOrderDeclaration.bounds).toBeUndefined(); + expect(cancelOrderDeclaration.refine).toBeUndefined(); +}); diff --git a/tests/simulator/example.test.ts b/tests/simulator/example.test.ts index 6ce37ab..5515d15 100644 --- a/tests/simulator/example.test.ts +++ b/tests/simulator/example.test.ts @@ -7,7 +7,10 @@ import { components, internal, } from "../../example/convex/_generated/api.js"; -import { placeOrderPermission } from "../../example/convex/ordering.js"; +import { + cancelOrderPermission, + placeOrderPermission, +} from "../../example/convex/ordering.js"; import { orderSummary } from "../../example/convex/orderSummary.js"; import { readOrdersPermission } from "../../example/convex/readModels.js"; import { @@ -45,6 +48,7 @@ async function caller( } const everything = [ placeOrderPermission, + cancelOrderPermission, receiveStockPermission, readOrdersPermission, "inventory.read", @@ -469,3 +473,254 @@ test( expect(await receipts(t)).toEqual([]); }, ); + +// Stock of 5 on sku-1, and order-1 placed for 3 of them over two lines, by a caller with every grant. +async function placedOrder(t: App) { + await activate(t); + const user = await caller(t, "user-1", everything); + await user.mutation(api.receiving.receiveStock, { + tenantId: "t-1", + input: { items: [{ stockItemId: "sku-1", quantity: 5 }] }, + }); + await user.mutation(api.ordering.placeOrder, { + tenantId: "t-1", + requestKey: "k-place", + input: { + orderId: "order-1", + lines: [line("sku-1", 2, 250), line("sku-1", 1, 999)], + }, + }); + return user; +} +const stockItem = (t: App) => + t.query(components.inventory.queries.stockItem.get, { + tenantId: "t-1", + streamId: "sku-1", + }); +const cancelCall = (requestKey: string, orderId = "order-1") => ({ + tenantId: "t-1", + requestKey, + input: { orderId }, +}); + +test( + name( + "PlaceOrder then CancelOrder: the order is cancelled, its stock item's totals are back to what they were, and its summary row says cancelled, in one commit", + ), + async () => { + const t = productionTest(); + const user = await placedOrder(t); + expect(await stockItem(t)).toMatchObject({ onHand: 5, allocated: 3 }); + const response = await user.mutation( + api.ordering.cancelOrder, + cancelCall("k-1"), + ); + const orderVersion = { + tenantId: "t-1", + contextId: "orders", + streamType: "order", + streamId: "order-1", + version: 2, + }; + expect(response).toMatchObject({ + kind: "applied", + replayed: false, + result: { + orderId: "order-1", + released: [{ stockItemId: "sku-1", quantity: 3 }], + }, + versions: [ + orderVersion, + { + ...orderVersion, + contextId: "inventory", + streamType: "stockItem", + streamId: "sku-1", + version: 3, + }, + ], + }); + expect(await stockItem(t)).toMatchObject({ + onHand: 5, + allocated: 0, + version: { version: 3 }, + }); + const order = await user.query(api.orderQueries.getOrder, { + tenantId: "t-1", + orderId: "order-1", + }); + expect(order).toMatchObject({ + orderId: "order-1", + status: "cancelled", + lines: [line("sku-1", 2, 250), line("sku-1", 1, 999)], + total: 1499, + version: orderVersion, + }); + expect(await summaries(t)).toMatchObject([ + { + key: "order-1", + generation: 1, + sourceVersions: [orderVersion], + status: "cancelled", + lineCount: 2, + total: 1499, + placedAt: order?.placedAt, + }, + ]); + const list = (status: "placed" | "cancelled") => + user.query(api.readModels.listOrderSummaries, { + tenantId: "t-1", + status, + paginationOpts: { cursor: null, numItems: 10 }, + }); + expect((await list("cancelled")).page).toMatchObject([ + { orderId: "order-1", status: "cancelled" }, + ]); + expect((await list("placed")).page).toEqual([]); + expect( + (await receipts(t)).filter((row) => row.requestKey === "k-1"), + ).toMatchObject([{ operationId: response.operationId }]); + }, +); + +test( + name( + "a second CancelOrder under a new key is rejected orderAlreadyCancelled, the same key is answered from its receipt, and an order never placed is rejected orderNotFound, and none of them stores anything", + ), + async () => { + const t = productionTest(); + const user = await placedOrder(t); + const first = await user.mutation( + api.ordering.cancelOrder, + cancelCall("k-1"), + ); + const before = { + stock: await stockItem(t), + summaries: await summaries(t), + receipts: await receipts(t), + }; + const after = async () => ({ + stock: await stockItem(t), + summaries: await summaries(t), + receipts: await receipts(t), + }); + expect( + await errorData( + user.mutation(api.ordering.cancelOrder, cancelCall("k-2")), + ), + ).toEqual({ + kind: "rejection", + code: "orderAlreadyCancelled", + commandType: "CancelOrder", + message: "The order is already cancelled", + }); + expect(await after()).toEqual(before); + expect( + await user.mutation(api.ordering.cancelOrder, cancelCall("k-1")), + ).toEqual({ + kind: "applied", + result: null, + operationId: first.operationId, + affected: first.affected, + versions: first.versions, + replayed: true, + }); + expect(await after()).toEqual(before); + expect( + await errorData( + user.mutation(api.ordering.cancelOrder, cancelCall("k-3", "order-2")), + ), + ).toEqual({ + kind: "rejection", + code: "orderNotFound", + commandType: "CancelOrder", + message: "The order does not exist", + }); + expect( + await user.query(api.orderQueries.getOrder, { + tenantId: "t-1", + orderId: "order-2", + }), + ).toBeNull(); + expect(await after()).toEqual(before); + }, +); + +test( + name( + "CancelOrder after Inventory released the order's units alone is rejected insufficientAllocation after Orders cancelled, and nothing of it is stored", + ), + async () => { + const t = productionTest(); + const user = await placedOrder(t); + // No command releases stock without cancelling its order: the setup calls the context's own + // operation, so the stock item's state stays the fold of its events. + await t.mutation(components.inventory.operations.release, { + tenantId: "t-1", + actor: operator, + operation: { + operationId: "setup-release", + causedBy: { kind: "command", commandType: "Setup" }, + }, + input: { + orderId: "order-1", + lines: [{ stockItemId: "sku-1", quantity: 3 }], + }, + }); + const stock = await stockItem(t); + expect(stock).toMatchObject({ allocated: 0, version: { version: 3 } }); + expect( + await errorData( + user.mutation(api.ordering.cancelOrder, cancelCall("k-1")), + ), + ).toEqual({ + kind: "rejection", + code: "insufficientAllocation", + commandType: "CancelOrder", + message: "Cannot release 3 when 0 are allocated", + details: { requested: 3, allocated: 0 }, + }); + expect( + await user.query(api.orderQueries.getOrder, { + tenantId: "t-1", + orderId: "order-1", + }), + ).toMatchObject({ status: "placed", version: { version: 1 } }); + expect(await stockItem(t)).toEqual(stock); + expect(await summaries(t)).toMatchObject([{ status: "placed" }]); + expect( + (await receipts(t)).filter((row) => row.requestKey === "k-1"), + ).toEqual([]); + }, +); + +test( + name( + "CancelOrder refuses a caller who holds every other grant but orders.cancel, before it reads, and stores nothing", + ), + async () => { + const t = productionTest(); + await placedOrder(t); + const other = await caller( + t, + "user-2", + everything.filter((permission) => permission !== cancelOrderPermission), + ); + expect( + await errorData( + other.mutation(api.ordering.cancelOrder, cancelCall("k-1")), + ), + ).toEqual({ + kind: "rejection", + code: "forbidden", + commandType: "CancelOrder", + message: "The caller may not run CancelOrder in this tenant", + details: { reason: "no_grant" }, + }); + expect(await stockItem(t)).toMatchObject({ allocated: 3 }); + expect(await summaries(t)).toMatchObject([{ status: "placed" }]); + expect( + (await receipts(t)).filter((row) => row.requestKey === "k-1"), + ).toEqual([]); + }, +); diff --git a/tests/types/command-declaration.renamed-handler-fails-build.test-d.ts b/tests/types/command-declaration.renamed-handler-fails-build.test-d.ts index 1415fc2..3d49369 100644 --- a/tests/types/command-declaration.renamed-handler-fails-build.test-d.ts +++ b/tests/types/command-declaration.renamed-handler-fails-build.test-d.ts @@ -64,7 +64,10 @@ bindExample(contract, (): { client?: ConvexHttpClient } => ({}), { }>(); }, "the change {change} is made": () => { - expectTypeOf().toEqualTypeOf<"submitOrder">(); + // The module's other public command, cancelOrder, keeps its name. + expectTypeOf().toEqualTypeOf< + "submitOrder" | "cancelOrder" + >(); }, "the check that fails first is {failsAt}": async ({ client }) => { // @ts-expect-error The caller's reference names an export the module no longer has. From cc00a3572c1b0c4531e6297102cd66da2f0a023e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:06:29 +0200 Subject: [PATCH 008/155] design: generations, the online rebuild, the gate, audit and the operator entry on paper A read model is installed through its first rebuild, declares its projection versions as a list, and keeps batch progress on a row no command reads. The rebuild's lifecycle is pinned with every transition and who makes it. The gate, the audit records and the one operator entry contract are pinned for the builds that follow. Co-Authored-By: Claude Fable 5.1 --- design/README.md | 16 +- ...ry.command-without-generation-fails.sdp.md | 4 +- ...ctivation-makes-read-model-writable.sdp.md | 20 +-- .../application/generation-registry.sdp.md | 79 ++++----- .../application/projection-contract.sdp.md | 30 ++-- ....committed-state-read-through-query.sdp.md | 2 +- ...read-models.committed-state-visible.sdp.md | 2 +- .../read-models.list-pages-by-cursor.sdp.md | 2 +- ...ild.online-rebuild-interrupt-resume.sdp.md | 8 +- design/specs/application/rebuild.sdp.md | 150 ++++++++++-------- .../rebuild.write-pause-rebuild-abort.sdp.md | 2 +- design/specs/application/restore.sdp.md | 8 +- design/specs/application/write-pause.sdp.md | 79 +++++---- design/specs/command/actor-and-scope.sdp.md | 21 ++- design/specs/command/command-pipeline.sdp.md | 4 +- .../command/tenancy-and-authority.sdp.md | 2 +- .../operations/baseline-operations.sdp.md | 25 +-- 17 files changed, 259 insertions(+), 195 deletions(-) diff --git a/design/README.md b/design/README.md index 96361e1..a92a9f3 100644 --- a/design/README.md +++ b/design/README.md @@ -178,7 +178,7 @@ Every row of the doc's acceptance table is an example Spec that `refines` and `v ## The extension register -An extension is a design claim the doc does not make. Each one is three things together: a bullet that starts with `[extension]` and cites its E-number, and a `[non-blocking] Extension E-n` open question in the Intent of the Spec that owns it. The register lists every E-number once. "Owned by" names the Specs that carry the open question and the ruling; other Specs cite the number as a cross-reference without repeating the question. Every extension is a provisional ruling the owner confirms, changes or rejects; none is blocking, because each has a working default. E-18, E-19, E-27 to E-29, E-48, E-49 and E-59 are unused; E-13 and E-39 were assigned by the integration round from the unassigned and the Package C range, E-14 by review round 2, and E-15 and E-16 by slice S0. +An extension is a design claim the doc does not make. Each one is three things together: a bullet that starts with `[extension]` and cites its E-number, and a `[non-blocking] Extension E-n` open question in the Intent of the Spec that owns it. The register lists every E-number once. "Owned by" names the Specs that carry the open question and the ruling; other Specs cite the number as a cross-reference without repeating the question. Every extension is a provisional ruling the owner confirms, changes or rejects; none is blocking, because each has a working default. E-18, E-19, E-27 to E-29 and E-59 are unused; E-13 and E-39 were assigned by the integration round from the unassigned and the Package C range, E-14 by review round 2, and E-15 and E-16 by slice S0. | E | Owned by | What the doc leaves open, and the option taken | |---|---|---| @@ -189,8 +189,8 @@ An extension is a design claim the doc does not make. Each one is three things t | E-5 | [`command.outcome-boundary`](specs/command/outcome-boundary.sdp.md) | `RejectionData` and `TransientData`; the platform codes `invalidInput`, `unauthenticated`, `forbidden`, `idempotencyConflict`, `staleVersion`, `entityExists`, `operationTooLarge`, `unsupportedContractVersion` and the transient `rateLimited`, `capacity`, `writePaused`; `normalizeThrown`, which turns a code the declaration's `rejections` does not list into a technical failure; a 16 KiB bound on `details` that `reject` measures; the engine's OCC-retry exhaustion as a retryable technical failure; the conditional dispatcher; a parent query's refusal as the same `RejectionData`, with the query's name in `commandType` and only the codes `unauthenticated` and `forbidden`; a use case's own rejection thrown as the kernel's bare `Rejection`, as a context throws one | | E-6 | [`command.actor-and-scope`](specs/command/actor-and-scope.sdp.md), [`command.tenancy-and-authority`](specs/command/tenancy-and-authority.sdp.md) | `Actor`, `ActorRef`, `TenantScope`, `CallerNamespace` (`public`, `service`, `worker`, `agent`, `system`), the `authorize` signature, the `grants` table; the tenant scope of an operation, a command and a query is the `tenantId` argument alone, with no separate scope argument; a parent query establishes and authorizes its actor through `authorizeQuery`, generic over the parent's data model, and its refusal is the command's `RejectionData` with the query's name in `commandType`; the worker's two modes as one `Authority` shape, `{ actor, scope, mode }` with `recheckDelegator` and `serviceAuthority`, stored on the obligation record and never restated; a derived command carries `worker`, and `system` is reserved for the platform's own maintenance chains and operator operations | | E-7 | [`command.command-declaration`](specs/command/command-declaration.sdp.md) | `CommandDeclaration`, `publicCommand` and `internalCommand`, the executor that takes the minted `OperationRef` and returns the kernel's outcome plus `streams`, the read-model binding with its `source` and read model, what generation emits, and no standing map from command type to declaration, because the type check at a typed caller is the check of Sc L2-4; the cost of two registered mutations per command and the single-export dispatcher alternative it beat | -| E-8 | [`application.generation-registry`](specs/application/generation-registry.sdp.md), [`application.write-pause`](specs/application/write-pause.sdp.md), [`application.rebuild`](specs/application/rebuild.sdp.md) | The `generations` table with seven states and their transitions, one building generation per read model, the `projectionMarkers` table; a read model's first generation made active by one explicit transition, the first activation, run with admin access and legal only when the read model has no generation row, a command that finds no generation to write and a query that finds no active generation each failing as a technical failure; `generationsToWrite` answering the active and the building generation as a list tagged by role; a 7-day rollback period in which the retired generation is kept but not written, and a rollback reopens it as `verifying` for one verify pass, with write-through as the recorded alternative; one generation per read model spanning every tenant, so the registry's two indexes lead with the read model name and not `tenantId`, a recorded deviation from Law 11's index rule; one gate document per deployment with scope keys, read by the pipeline's step 7, with no fence or drain of its own because optimistic concurrency and the generation's fence already serialize; a paused rebuild closes the view's source stream types, one `source:` scope per binding, so every writer of those streams is held; gate changes treated as mandatory audit | -| E-9 | [`application.projection-contract`](specs/application/projection-contract.sdp.md) | A per-entity read model declared as `ReadModel` with `name`, `table`, `rowBudgetBytes` and its `Projection` of `version`, `keyOf` and `project`, bound on a declaration with its source; `AggregateProjection` and `HistoryProjection`, the last folding events into the prior row for a view that depends on history across streams; `applyProjection` with modes `live-created`, `live-updated`, `backfill`, writing each generation by its role, where a live write to the active generation always lands and the rule that a newer row is never overwritten is the backfill path's, `applyAggregate` and `applyHistoryProjection`, called by the pipeline's step 9 and the rebuild's batches only; every row measured with `getConvexSize` against `rowBudgetBytes`; step 9 writes a per-entity read model from the DTO of each matching `streams` entry, never from the events; no check ties a generation to the projection version it records; row conventions `tenantId`, `generation`, `key`, `projectionVersion`, `sourceVersions`; the `by_key` index; the created flag and the appended events read from the operation outcome's `streams` entry | +| E-8 | [`application.generation-registry`](specs/application/generation-registry.sdp.md), [`application.write-pause`](specs/application/write-pause.sdp.md), [`application.rebuild`](specs/application/rebuild.sdp.md) | The `generations` table with seven states, every transition with who makes it, and one generation being built per read model; the `generationProgress` row that holds the cursor, the batch size and the counts, read by batches and operator entries only; the `projectionMarkers` table; a read model installed through its first rebuild, generation 1 starting as `building`, with no transition from nothing to active, a command that finds no generation to write and a query that finds no active generation each failing as a technical failure; `generationsToWrite` answering the active generation and the one being built as a list tagged by role, `active`, `building` or `verifying`; a 7-day rollback period in which the retired generation is kept and not written, and a rollback that is one verify pass, presence and absence, and then the switch of one write, a single mutation that patches both generation rows; one generation per read model spanning every tenant, so the registry's indexes lead with the read model name and not `tenantId`, a confirmed deviation from Law 11's index rule, and no rebuild for one tenant; one gate document per deployment with scope keys for the deployment, a tenant and a source stream type, read by the pipeline's step 7 after the restore door, with no fence or drain of its own; a paused rebuild closes the view's source stream types, one `source:` scope per binding; an exemption only for the generation a closed entry names; gate changes treated as mandatory audit; no acceptable length stated for a write pause | +| E-9 | [`application.projection-contract`](specs/application/projection-contract.sdp.md) | A per-entity read model declared as `ReadModel` with `name`, `table`, `rowBudgetBytes` and its `projections`, each a `Projection` of `version`, `keyOf` and `project`, bound on a declaration with its source; `AggregateProjection` and `HistoryProjection`, the last folding events into the prior row for a view that depends on history across streams; `applyProjection` with modes `live-created`, `live-updated`, `backfill`, writing each generation by its role, where a live write to the active generation always lands and the rule that a newer row is never overwritten is the backfill path's, `applyAggregate` and `applyHistoryProjection`, called by the pipeline's step 9 and the rebuild's batches only; every row measured with `getConvexSize` against `rowBudgetBytes`; step 9 writes a per-entity read model from the DTO of each matching `streams` entry, never from the events; a read model declares its projection versions as a list, the latest first, and each generation is written with the projection whose version its row records; row conventions `tenantId`, `generation`, `key`, `projectionVersion`, `sourceVersions`; the `by_key` index; the created flag and the appended events read from the operation outcome's `streams` entry | | E-10 | [`obligations.lifecycle-transitions`](specs/obligations/lifecycle-transitions.sdp.md), [`obligations.record-contract`](specs/obligations/record-contract.sdp.md) | The allowed transitions between the six states and who causes each; the `obligations` and `obligationRepairs` tables with seven and two indexes; a 64 KiB inline payload bound; ten late-evidence entries; two attention reasons beyond D13's four, `recoveryFailing` and `chainBound` | | E-11 | [`effects.claim-call-settle`](specs/effects/claim-call-settle.sdp.md) | The signatures of `claimAttempt`, `readForCall`, `callProvider`, `reconcileAttempt`, `settleAttempt`, `expireLease`; the `RepetitionPolicy` union with a fourth value `none` | | E-12 | [`constraints.events-stay-small`](specs/constraints/events-stay-small.sdp.md), [`context.event-envelope`](specs/context/event-envelope.sdp.md) | A 16 KiB payload bound and a 4,096-byte envelope bound measured with `getConvexSize` by the journal's `append`, so a fold of 500 stored events of at most 20,541 bytes each stays under 9.8 MiB under the 16 MiB read ceiling; the reserved `baseline` event's payload is exempt and bounded by its stream type's `budgetBytes`, and the stored baseline event is at most `budgetBytes + 4,096 + 61` bytes; the 1 second query timeout binds first and the first experiment measures it | @@ -213,17 +213,19 @@ An extension is a design claim the doc does not make. Each one is three things t | E-34 | [`command.idempotency-and-receipts`](specs/command/idempotency-and-receipts.sdp.md) | A default retry window of 7 days per command; lazy expiry at lookup; a per-tenant sweep of at most 1,000 rows; the tenant ID and the request key of at most 256 bytes of UTF-8, `limitIdLength`, checked at the pipeline's step 1 as `invalidInput`, and a declaration name above it a defect and a plain error; at most 1,000 affected refs per receipt | | E-35 | [`command.outcome-boundary`](specs/command/outcome-boundary.sdp.md) | The transient code `writePaused`, reserved for the write pause and confirmed by it | | E-36 | [`command.idempotency-and-receipts`](specs/command/idempotency-and-receipts.sdp.md) | The fingerprint is the SHA-256 digest of the canonical JSON, `convexToJson` with keys sorted, of the business input followed by the contract version; the outside facts the executor captures travel beside the input and are excluded, so a retry that captures a fresher fact is not a conflict | -| E-37 | [`command.tenancy-and-authority`](specs/command/tenancy-and-authority.sdp.md) | Identity to actor mapping from `tokenIdentifier`, `subject` and `issuer`; grants written by the plain helpers `insertGrant` and `revokeGrant`, a tenant's first grant by an operator with admin access; revocation by deleting the grant row, with no cache; a 500-grant read bound | +| E-37 | [`command.tenancy-and-authority`](specs/command/tenancy-and-authority.sdp.md) | Identity to actor mapping from `tokenIdentifier`, `subject` and `issuer`; grants written by the plain helpers `insertGrant` and `revokeGrant`, a tenant's first grant created with admin access through the composition's own internal grant mutation, with no grant function, bootstrap, first-admin flow or invitation in the library; revocation by deleting the grant row, with no cache; a 500-grant read bound | | E-38 | [`command.command-declaration`](specs/command/command-declaration.sdp.md) | The declaration also carries `contractVersion`, `retention`, `irreversible`, `admission`, `bounds`, `rejections`, `audit` and `writes`; `bounds` is the one input bound, checked at the pipeline's step 1; `writes` names the context and stream type pairs the executor may write, from which step 7 derives the write pause's source scopes and which step 9 proves; the declaration has no reactions field | | E-39 | [`command.idempotency-and-receipts`](specs/command/idempotency-and-receipts.sdp.md) | The second UI submit of a create is answered with the rejection `entityExists`, not an idempotent success, because the UI path writes no receipt to replay from and the client holds the entity ID | -| E-40 | [`application.rebuild`](specs/application/rebuild.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | A batch is one mutation, one cursor, at most 100 entities capped by the context's byte-derived page bound, a design number not attributed to the component; `@convex-dev/migrations` drives parent-table batches and a self-scheduled internal mutation drives context-enumerated ones until Probe 6 shows the component can; all-tenants iteration over the parent's tenant list | +| E-40 | [`application.rebuild`](specs/application/rebuild.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | A batch is one mutation with one cursor on the progress row, at most 100 entities capped by the row budget and by the context's byte-derived page bound, a design number not attributed to the component; a self-scheduled internal mutation drives the context-enumerated batches, the backfill and the verify pass, and `@convex-dev/migrations` is the driver of a parent-table batch, the purge, once Probe 6 shows it fits, the purge being self-scheduled until then; one batch names one tenant, taken from the tenant list; a fence that every interrupt, resume, rollback, abort and purge start raises | | E-41 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | The caller-driven import command with an `imports` progress row; the input bound itself is the declaration's `bounds.maxItems` under E-38 | | E-42 | [`application.restore`](specs/application/restore.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | Four checks (journal, domain, read model, receipt) as checkpointed batches, the stream check bounded through the `rebuild` query's `maxEvents` so its worst case is one fold, and a refused fold counted as a journal finding; a fifth step that inventories the closed gate entries, the generations in flight and the running imports the snapshot restored; the `restoreRuns` record; an accepted branch that resumes every stranded generation, proven by the generation row's fence; `MAINTENANCE_MODE=restore` as the writer door, read by the write pause's check before the gate document and cleared before the durable profile's schedule rebuild runs | -| E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | +| E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed, with `AuditRecordInput` and the source of each field in the command; the `operatorAudit` table for the gate changes an operator entry makes; Convex's own audit logging recorded as the do-nothing option; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | | E-44 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | `BusinessFailurePolicy` (`carry` or `reject`) per context call, applied in the executor body through the pinned helper `carryOrReject` | | E-45 | [`application.read-models`](specs/application/read-models.sdp.md) | At most 8 component reads per parent query and 4 read-model rows written per command, counted as the rows step 9 inserts, replaces or deletes and checked after each `streams` entry is applied, above which step 9 throws a plain error | | E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a provisional maximum of 100 lines; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` calls Orders first; the order allocation history view that Sc L2-6 builds | | E-47 | [`application.first-experiment`](specs/application/first-experiment.sdp.md) | The `Measurement` record, one per measured run, and how each count is captured: commits from the backend's function log, documents and bytes from the `usageStats` of the command's top-level completion record, the calls per context from a pure test of the use case's executor because the log holds no record of a component call inside a mutation, and a count the backend does not expose recorded as a gap; the documents one `PlaceOrder` reads, expected near N + 6 and counted 3N + 5 on the pinned backend, left for the measurement run to explain | +| E-48 | [`command.actor-and-scope`](specs/command/actor-and-scope.sdp.md) | The operator entry: an internal function that only admin access reaches, a required `operator` string of 1 to 512 bytes checked by `assertOperator`, no grant read, and the stated operator recorded on the generation row, the gate's entry and audit record, or the restore's run record; a stated operator proves no identity | +| E-49 | [`command.actor-and-scope`](specs/command/actor-and-scope.sdp.md) | The tenant list: a `tenants` table written by `insertGrant` with a tenant's first grant and never deleted, read one tenant at a time through `nextTenant` in tenant-ID order by every pass over the whole deployment | | E-50 | [`obligations.obligation-module`](specs/obligations/obligation-module.sdp.md) | The module is a parent-side library plus two parent tables with a static handler registry, not a component | | E-51 | [`obligations.lifecycle-transitions`](specs/obligations/lifecycle-transitions.sdp.md), [`obligations.local-reaction-wrapper`](specs/obligations/local-reaction-wrapper.sdp.md), [`effects.external-effects`](specs/effects/external-effects.sdp.md) | A body's business failure settles `succeeded` with event evidence; a thrown rejection or a declined report settles `cancelled` with `businessRejection` evidence; backoff 1 s times 2 capped at 1 h; 8 attempts by default; a transient `writePaused` or `capacity` refusal defers the same attempt after `retryAfterMs` or 60 s without counting it | | E-52 | [`obligations.sweeper`](specs/obligations/sweeper.sdp.md) | A cron every 60 s, a grace of 120 s, 100 candidates per scan, 5 rearms, per-candidate nested-mutation isolation, plus a 500 ms elapsed guard by `performance.now()`; the 100 bound provisional on Probe 3; dispatch state read from the system row's lowercase `state.kind` with the failed row's error kept on the rearmed record | @@ -307,7 +309,7 @@ The Layer 2 design added two more for the same reason, and nine examples that ve |---|---|---| | L2-2 | `spec:application.read-models.committed-state-read-through-query` | the row says "a query or subscription" | | L2-4 | `spec:command.command-declaration.broken-argument-contract-fails-build` | the row says "rename a typed handler or break its argument contract" | -| none | `spec:application.generation-registry.first-activation-makes-read-model-writable` | verifies the registry's first activation, under E-8 | +| none | `spec:application.generation-registry.first-activation-makes-read-model-writable` | verifies that a read model is installed through its first rebuild, under E-8 | | none | `spec:application.generation-registry.command-without-generation-fails` | verifies that a command whose read model has no generation fails, under E-8 | | none | `spec:application.read-models.query-refused-before-disclosure` | verifies Law 5 for a parent query called with no identity | | none | `spec:application.read-models.query-refused-without-grant` | verifies Law 5 for a parent query called with no grant in the tenant | diff --git a/design/specs/application/generation-registry.command-without-generation-fails.sdp.md b/design/specs/application/generation-registry.command-without-generation-fails.sdp.md index 297533f..37f649e 100644 --- a/design/specs/application/generation-registry.command-without-generation-fails.sdp.md +++ b/design/specs/application/generation-registry.command-without-generation-fails.sdp.md @@ -25,6 +25,6 @@ And the number of stored receipts, events and state changes is {stored: 0} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- The test creates stock through `ReceiveStock`, which declares no read model, and sends `PlaceOrder` before any first activation of the order summary. +- The test creates stock through `ReceiveStock`, which declares no read model, and sends `PlaceOrder` before any generation of the order summary is started. - The test asserts that the caller's error is not a `ConvexError` of the outcome boundary, that the function log's record of the failure names the order summary, and that no receipt, no order stream, no order event and no change to the stock exists for the operation. -- The test then runs the first activation, re-sends the same command with the same request key and asserts that it is applied. +- The test then starts generation 1 of the order summary, re-sends the same command with the same request key while the generation is still being built, and asserts that it is applied and that its row is in generation 1. diff --git a/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md b/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md index 5d78a6d..e82b5a6 100644 --- a/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md +++ b/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md @@ -7,24 +7,26 @@ relations: refines: spec:application.generation-registry verifies: spec:application.generation-registry --- -# The first activation makes a read model writable +# A read model is installed through its first rebuild -E-8 · native tier · production composition · no acceptance row; the example verifies the first activation, an extension under E-8. +E-8 · native tier · production composition · no acceptance row; the example verifies that a read model's first generation is built like any other and serves the subjects that already exist, an extension under E-8. ## Intent -- outcome: A read model with no generation gets generation 1 as active through one explicit transition, which a second call refuses. (E-8, D9) +- outcome: Generation 1 starts as building, covers the subjects that existed before it, and is active only after it is verified and switched. (E-8, D9) ```gwt -Given a read model with {generationRows: 0} generation rows -When an operator runs the first activation with admin access +Given a read model with {generationRows: 0} generation rows and {orders: 3} subjects that already have history +When an operator starts a generation, its batches finish and the operator switches it Then generation {generation: 1} of the read model is {state: "active"} -And a second first activation is {second: "refused"} +And the read model holds {rows: 3} rows for the subjects that existed before it +And a second start while generation 1 is being built is {second: "refused"} ``` ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- The test runs the production composition's first activation for the order summary with admin access, then reads the `generations` table with admin access and asserts one row for the read model, generation 1, `active`, with the order summary projection's version. -- The test runs the first activation a second time and asserts that its error names the read model, generation 1 and the state `active`, and that the table still holds one row. -- The test then creates stock through `ReceiveStock`, sends `PlaceOrder`, and asserts that the order's summary row exists in generation 1. +- The test places three orders through a command that declares no read model, or writes them before the read model is declared, then runs `internal.rebuild.startGeneration` for the order summary with admin access and a stated operator, and asserts one row in `generations`, generation 1, `building`, with the order summary's latest projection version and the operator as `startedBy`. +- The test runs `startGeneration` again before the first generation is switched and asserts that its error names the read model, generation 1 and its state, and that the table still holds one row. +- The test waits until the row is `verified`, asserts that a query over the read model still fails for want of an active generation, runs `switchGeneration`, and asserts that the row is `active`, that no row is `retired`, and that the list query returns the three orders. +- The test runs the same sequence on a backend with no subject and asserts that the progress row shows one backfill batch and one verify batch before `verified`. diff --git a/design/specs/application/generation-registry.sdp.md b/design/specs/application/generation-registry.sdp.md index 81ecaf7..7e5217a 100644 --- a/design/specs/application/generation-registry.sdp.md +++ b/design/specs/application/generation-registry.sdp.md @@ -17,72 +17,79 @@ relations: --- # Generation registry -Story · Detail: full · Traces: D8, D9, D11, D19, F1, F12, F13, Law 6, Law 11, E-8, E-9, E-15, E-40, E-42. +Story · Detail: full · Traces: D8, D9, D11, D19, F1, F12, F13, Law 6, Law 11, E-8, E-9, E-40, E-42, E-48, E-49. -The registry is the parent table that records every generation of every read model, its state and its progress, plus the per-entity marker table that aggregate read models use. A generation is a build of the projection's code, which is the same for every tenant, so one generation row spans every tenant of the deployment and its batches iterate the tenants; there is no generation of one tenant, because a switch that made a one-tenant build the active generation would leave every other tenant reading an empty read model. A use case reads the registry once per read model it maintains to learn which generations to write; a query reads it once to learn which generation to read; the rebuild owns its rows. The registry is data that changes only during a rebuild, so it is not an application-wide counter in the write path. +The registry is two parent tables. The `generations` table records every generation of every read model with its state, its projection version and its fence, and it is the row a command and a query read. The `generationProgress` table holds, per generation, the cursor, the batch size and the counts of the batches that build, verify and purge it, and only a batch and an operator entry read it. A generation is a build of the projection's code, which is the same for every tenant, so one generation row spans every tenant of the deployment and its batches iterate the tenants; there is no generation of one tenant and no rebuild for one tenant. A use case reads the registry once per read model it maintains to learn which generations to write; a query reads it once to learn which generation to read; the rebuild owns the rows. The registry changes only at a transition of a generation, so it is not an application-wide counter in the write path. ## Intent -- outcome: Pin the generations table and its states, the per-entity marker document, and the reads a command and a query make against them (D9, E-8) -- value: One table answers which generation is active, which is building and which is still inside its rollback period, so live update, backfill, queries and operators agree without a second store (D9) -- risk: A registry read sits in every command that maintains a read model, one indexed read per read model; the first experiment counts it as a document read (D9, Sc L2-3) +- outcome: Pin the generations table and its states, the progress row, every transition with who may make it, the per-entity marker document, and the reads a command and a query make against them (D9, E-8) +- value: One table answers which generation is active, which is being built and which is still inside its rollback period, so live update, backfill, queries and operators agree without a second store (D9) +- risk: A registry read sits in every command that maintains a read model, at most two indexed reads per read model; every transition of a generation changes a row those reads cover, so each transition re-runs the commands in flight on that read model once (D9, F1, Sc L2-3) +- risk: A projection version stays in the code for as long as a generation that records it is written or may be rolled back to (E-9, D9) - assumption: Mutations are serializable under optimistic concurrency, so a switch and a concurrent command that read the registry serialize (F1) ### Open questions -- [non-blocking] Extension E-8: the doc names a generation registry, a marker and a gate without their shapes, and says to keep the old generation for a rollback period without saying whether live commands keep writing it; the option taken here is the `generations` and `projectionMarkers` tables below with seven states, a rollback period of 7 days, one building generation per read model at a time, and a retired generation that is kept but not written, so that a rollback reopens it as `verifying` and one verify pass repairs the rows changed since the switch before it is active again; the other reading is write-through, where live commands keep writing the retired generation for the whole period so that rollback is one write, at the cost of a third point read and write per read model per command for 7 days after every cutover; the cheaper steady path is taken and the owner rules whether the product needs instant rollback (E-8, D9, Sc L2-5) -- [non-blocking] Extension E-8, reach of a generation: the doc says nothing about tenants and rebuild; the option taken here is one generation per read model spanning every tenant, so the registry has no tenant dimension, one row is active for all tenants and the batches iterate the tenant list; the other reading is one registry per read model and tenant, keyed by `readModel`, `tenantId` and `generation`, which lets one tenant rebuild while another keeps reading and lets a write pause close one tenant at a time, at the cost of one generation row, switch, rollback and purge per tenant, a projection-version rollout that is one generation per tenant, and a third index field on every registry read; the deployment-wide reading is taken because a generation is a build of the projection's code and the owner rules whether per-tenant rebuilds are a product need; a consequence the owner confirms with it is that the `generations` table is operational data whose scope is the whole deployment, not tenant data, so its two indexes `by_read_model` and `by_read_model_state` lead with the read model name rather than `tenantId`, a recorded deviation from Law 11's index rule and the review rubric's item 25, of the same kind the obligation module records under E-57, while the marker table, which is tenant data, leads with the tenant (E-8, D9, D11, Law 11, E-57) -- [non-blocking] Extension E-42, as it touches the registry: a restore brings back a generation row at its checkpoint and drops the scheduled batch that was driving it, so the row's fence at inventory time is the restore's proof that the generation is still stranded, and `resumeChain` bumping it is the proof that it is not; the owner confirms that the fence carries that meaning (E-42, F12) -- [non-blocking] Extension E-8, the first generation: the doc says how a generation is rebuilt and not how a read model's first generation comes to exist; the option taken here is one explicit transition, the first activation, run with admin access before the first command that writes the read model, legal only when the read model has no generation row, with a command that finds no generation to write failing as a technical failure and a query that finds no active generation failing the same way; the owner has not ruled (E-8, D9, D19) -- [non-blocking] Extension E-40: the batch runner writes its cursor and counts on the generation row; the migrations component keeps its own state in its own table when it drives a parent-table batch, and the two are reconciled by the reviewed check below (E-40, F17) +- [non-blocking] Extension E-8: the doc names a generation registry, a marker and a gate without their shapes, and says to keep the old generation for a rollback period without saying whether live commands keep writing it; the option taken here is the `generations` and `projectionMarkers` tables below with seven states, a rollback period of 7 days, one generation being built per read model at a time, and a retired generation that is kept and not written, so that a rollback reopens it as `verifying`, one verify pass repairs what the writes since its retirement changed, and the ordinary switch makes it active again; a rollback is therefore a verify pass and then a switch of one write, never one write by itself (E-8, D9, Sc L2-5) +- [non-blocking] Extension E-8, reach of a generation: the doc says nothing about tenants and rebuild; one generation per read model spans every tenant, the registry has no tenant dimension, one row is active for all tenants and the batches iterate the tenant list of `spec:command.actor-and-scope`; the registry is operational data of the deployment, so its indexes lead with the read model name and not with `tenantId`, a confirmed deviation from Law 11's index rule of the same kind the obligation module records under E-57, while every table that holds tenant data, the read-model rows and the markers among them, leads with the tenant; a rebuild for one tenant is not offered (E-8, D9, D11, Law 11, E-57) +- [non-blocking] Extension E-42, as it touches the registry: a restore brings back a generation row and its progress row at their checkpoint and drops the scheduled batch that was driving them, so the row's fence at inventory time is the restore's proof that the generation is still stranded, and `resumeChain` bumping it is the proof that it is not (E-42, F12) +- [non-blocking] Extension E-8, the first generation: the doc says how a generation is rebuilt and not how a read model's first generation comes to exist; the option taken here is that a read model is installed through its first rebuild, generation 1 starting as `building` like any other, so a read model added to a deployment that has history serves that history from its first switch and no transition claims coverage it did not build; a command that finds no generation to write fails as a technical failure and a query that finds no active generation fails the same way (E-8, D9, D19) +- [non-blocking] Extension E-40, the checkpoint: the cursor, the batch size and the counts live on the progress row and not on the generation row, because a batch that wrote its checkpoint on the row every command reads would re-run every command in flight on the read model once per batch; the progress row is the one checkpoint of every batch whatever drives it (E-40, F1) ## Contract -- Every generation of a read model has one row keyed by read model name and generation number, and generation numbers increase by one per read model (E-8, D9) +- Every generation of a read model has one row keyed by read model name and generation number, and generation numbers increase by one per read model, starting at 1 (E-8, D9) - A generation is in exactly one of `building`, `verifying`, `verified`, `active`, `retired`, `aborted` or `purged` (E-8, D9) -- At most one generation per read model is `active`, and at most one is in `building`, `verifying` or `verified` (E-8, D9) -- The allowed transitions are none to active for a read model's first generation, by the first activation, building to verifying, verifying to verified, verified to active, active to retired, retired to verifying by rollback, retired to purged, and building, verifying or verified to aborted, aborted to purged; no other transition exists (E-8, D9) -- A command that maintains a read model writes the active generation and the building, verifying or verified generation if one exists; a retired generation is kept unchanged until `retireAfter` and never written, and a rollback reopens it as `verifying` so that live commands write it again while its verify pass repairs it (D9, E-8) +- At most one generation per read model is `active`, and at most one is in `building`, `verifying` or `verified`; these three states are the in-flight states (E-8, D9) +- The allowed transitions are none to building by `startGeneration`, building to verifying by the backfill batch that ends the enumeration, verifying to verified by the verify batch that ends its pass, verified to active by `switchGeneration`, active to retired by the same `switchGeneration`, retired to verifying by `rollbackGeneration`, building, verifying or verified to aborted by `abortGeneration`, and retired or aborted to purged by the purge batch that ends its pass; no other transition exists, and none leads from nothing to active (E-8, D9) +- An operator entry of `spec:application.rebuild` makes the transitions named after it, a scheduled batch makes the other three, and nothing else writes a generation row (E-8, E-48, D19) +- [extension] A read model's first generation is generation 1 in `building`, started by `startGeneration` like any other, and the read model has an active generation only after that generation is verified and switched; on a deployment with no subject the build is one backfill batch and one verify batch (E-8, D9) +- A command that maintains a read model writes the active generation and the in-flight generation if one exists; a retired generation is kept unchanged until it is purged and never written, and a rollback reopens it as `verifying` so that live commands write it again while its verify pass repairs it (D9, E-8) +- [extension] Each generation is written with the projection whose version its row records, on the live path and by its batches alike, so the rows of one generation are one truth of one projection version (E-9, E-8, D8) - A query reads the active generation and nothing else; a query over a read model that has no active generation throws a plain error that names the read model, a technical failure, and never answers an empty page (D9, E-8, Law 6) -- The generation row carries the backfill cursor, the verify cursor, the fence and the counts, and a batch writes them in the same mutation as its rows, so the row is the checkpoint (D9, E-40) +- [extension] The generation row carries the state, the projection version, the fence and who started and last changed it, and nothing a batch writes on its healthy path; the progress row carries the cursor, the batch size and the counts, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) - [extension] A generation spans every tenant of the deployment and the registry row names none; the tenant is named on every read-model row and marker, on every context call a batch makes, and in the batch cursor, so no read or write of tenant data runs with an absent tenant as a wildcard, and no generation can be switched for one tenant alone (E-8, D11, Law 11) - An aggregate read model keeps one marker per entity per generation with the contribution last recorded and the source versions it was computed from, and every aggregate write reads and writes the marker in the same mutation as the aggregate row; the marker's `sourceVersions` are compared like a row's, so a DTO older than the marker never adjusts the sum (D9, E-8, E-9) -- [extension] A generation in `building` or `verifying` has exactly one scheduled batch driving it, identified by the row's fence; a restore brings the row back at its checkpoint and drops that batch with every pending scheduled function, so the generation is stranded until `resumeChain` bumps the fence and reschedules from the saved cursor, which the restore procedure's pending-work inventory and accepted branch drive for every such row (E-42, E-8, F12, D19) +- [extension] A generation whose progress row is in the phase `backfill`, `verify` or `purge` has at most one scheduled batch driving it, identified by the row's fence; a restore brings the rows back at their checkpoint and drops that batch with every pending scheduled function, so the generation is stranded until `resumeChain` bumps the fence and reschedules from the saved cursor, which the restore procedure's pending-work inventory and accepted branch drive for every such row (E-42, E-8, F12, D19) - The registry is read, never incremented, in the command path; it is not an application-wide counter (D2, E-8) -- Registry and marker rows are operational data owned by the parent; they are never written by a context (D2, E-8) -- [extension] A read model's first generation becomes `active` through one explicit transition, the first activation, which is legal only when the read model has no generation row in any state; it inserts generation 1 as `active` with the declared projection's version and reads no source, so it claims nothing about events that already exist, and a read model added to a deployment that has history is a rebuild's work (E-8, D9) -- [extension] The library registers no function for the first activation: the application registers one internal mutation over the helper, an operator runs it with admin access once per read model before the first command that writes it, and a native test does the same as setup (E-8, E-15, D19) +- Registry, progress and marker rows are operational data owned by the parent; they are never written by a context (D2, E-8) - [extension] A command whose declared read model has no generation to write fails as a technical failure, a plain error that names the read model, so a command never commits without its read model (E-8, D8, Law 6) +- [extension] The operator a generation row records is the string the operator entry was given, as `spec:command.actor-and-scope` rules it: `startedBy` from `startGeneration` and `changedBy` from the last operator entry that changed the row (E-48, D19) ## Design -Tables live in the parent's `schema.ts`. `actorValidator` is the actor shape the command pipeline establishes. `streamVersionValidator` is the kernel's validator for `StreamVersion`. The registry has no `tenantId` because every generation spans all tenants; the marker table is tenant data and leads with the tenant. +Tables live in the parent's `schema.ts`, spread from the library's `readModelTables`. `streamVersionValidator` is the kernel's validator for `StreamVersion`. The registry has no `tenantId` because every generation spans all tenants; the marker table is tenant data and leads with the tenant. Every refusal below is a plain `Error` with the message given. - validatorGenerationState: `const generationStateValidator = v.union(v.literal("building"), v.literal("verifying"), v.literal("verified"), v.literal("active"), v.literal("retired"), v.literal("aborted"), v.literal("purged"))` (E-8, D9) -- validatorBatchCursor: `const batchCursorValidator = v.object({ tenantId: v.string(), pageCursor: v.union(v.string(), v.null()), streamId: v.optional(v.string()), eventCursor: v.optional(v.union(v.string(), v.null())) })` the position of a backfill or verify pass, the tenant being iterated and the `paginator` cursor within it, `null` at the start of a tenant, plus, for a history view, the stream whose history is being paged and the `history` cursor within it; the rebuild's `tenantIteration` moves it to the next tenant when a page reports `isDone` (E-8, E-40, Law 11) -- tableGenerations: `generations: defineTable({ readModel: v.string(), generation: v.number(), projectionVersion: v.number(), state: generationStateValidator, pauseRequired: v.boolean(), batchSize: v.number(), fence: v.number(), cursor: v.optional(batchCursorValidator), verifyCursor: v.optional(batchCursorValidator), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), startedAt: v.number(), startedBy: actorValidator, switchedAt: v.optional(v.number()), retireAfter: v.optional(v.number()), lastError: v.optional(v.string()), updatedAt: v.number() })` (E-8, D9, F13) +- tableGenerations: `generations: defineTable({ readModel: v.string(), generation: v.number(), projectionVersion: v.number(), state: generationStateValidator, pauseRequired: v.boolean(), fence: v.number(), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), switchedAt: v.optional(v.number()), retiredAt: v.optional(v.number()), retireAfter: v.optional(v.number()) })` where `changedAt` and `changedBy` are the time and the operator of the last operator entry that changed the row, `switchedAt` is when the generation last became active, and `retiredAt` and `retireAfter` are set while it is `retired` and absent otherwise (E-8, E-48, D9, F13) - indexGenerationsByReadModel: `.index("by_read_model", ["readModel", "generation"])` (E-8) -- indexGenerationsByReadModelUse: the first activation's check that the read model has no row, the next generation number at `startGeneration`, and the operator listing (E-8, D9) +- indexGenerationsByReadModelUse: the next generation number at `startGeneration`, read as the last row in descending order, and the operator listing `getGenerations` (E-8, D9) - indexGenerationsByReadModelState: `.index("by_read_model_state", ["readModel", "state"])` (E-8) -- indexGenerationsByReadModelStateUse: `generationsToWrite` in every command that maintains the read model, and `activeGeneration` in every query over it (D9, E-8) +- indexGenerationsByReadModelStateUse: `generationsToWrite` in every command that maintains the read model, `activeGeneration` in every query over it, and the in-flight check of `startGeneration`, `switchGeneration` and `rollbackGeneration` (D9, E-8) +- validatorBatchCursor: `const batchCursorValidator = v.object({ tenantId: v.string(), pageCursor: v.union(v.string(), v.null()), streamId: v.optional(v.string()), eventCursor: v.optional(v.union(v.string(), v.null())) })` the position of a pass, the tenant being iterated and the `paginator` cursor within it, `null` at the start of a tenant; `streamId` and `eventCursor` are set only by the batches of a history view (E-8, E-40, Law 11) +- validatorProgressPhase: `const progressPhaseValidator = v.union(v.literal("backfill"), v.literal("verify"), v.literal("purge"), v.literal("idle"))` naming the pass a batch chain is driving, `idle` when none is (E-40) +- tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), phase: progressPhaseValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position of the pass `phase` names and `null` before that pass has read its first tenant; `batchesDone` counts every committed batch of the generation, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted (E-40, D9, F13) +- indexProgressByGeneration: `.index("by_generation", ["generationId"])` (E-40) +- indexProgressByGenerationUse: the one read a batch and an operator entry make to find a generation's progress row; no command and no query over a read model reads the table (E-40, F1) - tableProjectionMarkers: `projectionMarkers: defineTable({ tenantId: v.string(), readModel: v.string(), generation: v.number(), entityKey: v.string(), aggregateKey: v.string(), contribution: v.number(), sourceVersions: v.array(streamVersionValidator) })` (E-8, D9, Law 11) - indexMarkersByEntity: `.index("by_entity", ["tenantId", "readModel", "generation", "entityKey"])` (E-8) -- indexMarkersByEntityUse: point read by `applyAggregate` before every aggregate write, and the purge batch's range for a generation (D9, E-8) -- typeRegistryReader: `type RegistryReader = { db: GenericDatabaseReader }` and `type RegistryWriter = { db: GenericDatabaseWriter }`, the part of a ctx the registry's helpers use, where `ReadModelDataModel` is the data model of `readModelTables`, the schema fragment that holds `generations`; every helper that takes them is generic over the composition's data model, because a database reader is not covariant in its data model (E-8) -- fnGenerationsToWrite: `generationsToWrite(ctx: RegistryReader, readModel: string): Promise` making at most two indexed reads, the ranges of `by_read_model_state` from `active` to `building` and from `verified` to `verifying`, throwing a plain error when they hold two active generations or two in `building`, `verifying` or `verified`, and returning one entry per generation a command writes, the active generation with `role: "active"` first, then the one in `building`, `verifying` or `verified` with `role: "building"`, each with its `generation` and `projectionVersion`, and an empty list when the read model has no generation to write; the answer is the same for every tenant, so a command on any tenant writes the same generations (E-8, D9) -- fnActivateFirstGeneration: `activateFirstGeneration(ctx: RegistryWriter, readModel: ReadModel, startedBy: Actor): Promise` reads `by_read_model` for any row of the read model and throws a plain error that names the existing generation and its state when one exists; otherwise it inserts generation 1 with `state: "active"`, `projectionVersion: readModel.projection.version`, `pauseRequired: false`, `batchSize`, `fence` and the four counts at 0, `startedAt`, `switchedAt` and `updatedAt` at the mutation's `Date.now()` and `startedBy` as given, and returns 1 (E-8, D9) -- firstActivationEntry: `export const activate = internalMutation({ args: { readModel: v.string(), startedBy: actorValidator }, returns: v.number(), handler })` in the application, whose handler finds the read model by name in the application's list of declared read models and throws a plain error for a name the deployment does not declare; `startedBy` is the actor its trusted caller states, as a command's internal entry takes its actor (E-8, D19, D11) -- fnActiveGeneration: `activeGeneration(ctx: RegistryReader, readModel: string): Promise` making one indexed read and answering `undefined` when the read model has no active generation, which a query over the read model then turns into a plain error that names it; the answer is the same for every tenant, and the query narrows the rows it reads by its own `tenantId` argument (E-8, D9, Law 11) +- indexMarkersByEntityUse: point read by `applyAggregate` before every aggregate write (D9, E-8) +- typeRegistryReader: `type RegistryReader = { db: GenericDatabaseReader }` and `type RegistryWriter = { db: GenericDatabaseWriter }`, the part of a ctx the registry's helpers use, where `ReadModelDataModel` is the data model of `readModelTables`, the schema fragment that holds `generations`, `generationProgress` and `projectionMarkers`; every helper that takes them is generic over the composition's data model, because a database reader is not covariant in its data model (E-8) +- typeWritableGeneration: `type WritableGeneration = { role: "active" | "building" | "verifying"; generation: number; projectionVersion: number }` where the role is `active` for the state `active`, `building` for the state `building`, and `verifying` for the states `verifying` and `verified` (E-8, E-9, D9) +- fnGenerationsToWrite: `generationsToWrite(ctx: RegistryReader, readModel: string): Promise` making at most two indexed reads, the ranges of `by_read_model_state` from `active` to `building` and from `verified` to `verifying`, throwing `"Read model " + readModel + " has " + a + " active generations and " + b + " in building, verifying or verified"` when they hold two active generations or two in-flight ones, and returning one entry per generation a command writes, the active generation first when there is one, then the in-flight one, and an empty list when the read model has neither; the answer is the same for every tenant (E-8, D9) +- fnActiveGeneration: `activeGeneration(ctx: RegistryReader, readModel: string): Promise` making one indexed read and answering `undefined` when the read model has no active generation, which a query over the read model then turns into the plain error `"The read model " + readModel + " has no active generation"`; the answer is the same for every tenant, and the query narrows the rows it reads by its own `tenantId` argument (E-8, D9, Law 11) +- errorNoGenerationToWrite: the pipeline's step 9 throws `commandType + " writes the read model " + readModel + ", which has no generation to write"` when `generationsToWrite` answers an empty list, which is the case before `startGeneration` and after the only generation was aborted (E-8, D8, Law 6) - markerVersionCompare: `applyAggregate` treats the marker's `sourceVersions` as a row's: a marker whose versions are not older than the input's makes the write `skipped-newer` and leaves the sum alone, so a stale DTO on either path never adjusts an aggregate twice or backwards (E-9, E-8, D9, F1) -- fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(generationDoc), handler })` for the operator (E-8, D19) +- transitionCost: a transition patches one generation row, or two in the one write of a switch that retires an active generation, a single mutation, and every command in flight on the read model that read the registry before it re-runs once; a batch writes a generation row only in the mutation that ends its pass, so a build costs the commands three such re-runs, at the start, at the end of the backfill and at the end of the verify pass, plus one per operator entry (E-40, F1, D9) - limitRegistryReadsPerCommand: at most 2 indexed reads per read model per command, because the pipeline's step 9 calls `generationsToWrite` once per read model name however many bindings name it, and one per query (E-8, F13) -- limitRollbackPeriod: 7 days by default; `switchGeneration` takes an override, and `rollbackGeneration` is refused after `retireAfter` because the rows may be purged (E-8, D9) -- limitMarkerCount: one marker per entity per generation, purged with the generation; a read model with N entities costs N markers per generation (E-8, F13) -- migrationsState: when the migrations component drives a parent-table batch, its own status table holds the batch cursor and the generation row records only the state change; the reviewed check confirms the two never disagree on `state` (E-40, F17) +- limitRollbackPeriod: 7 days by default, 604,800,000 ms; `switchGeneration` takes an override between 0 and 2,592,000,000 ms, and `rollbackGeneration` is refused once `retireAfter` has passed because the rows may be purged (E-8, D9) +- limitMarkerCount: one marker per entity per generation; a read model with N entities costs N markers per generation (E-8, F13) ## Verification — reviewed -- A reviewer confirms that every transition in the rebuild workflow is one of the transitions listed here and that no function sets a state outside them. -- A reviewer confirms that `generationsToWrite` and `activeGeneration` are the only readers of the registry in the command and query paths, and that neither writes it. +- A reviewer confirms that every transition in the rebuild workflow is one of the transitions listed here, that no function sets a state outside them, and that no function inserts a generation in a state other than `building`. +- A reviewer confirms that `generationsToWrite` and `activeGeneration` are the only readers of the registry in the command and query paths, that neither writes it, and that neither reads `generationProgress`. +- A reviewer confirms that no batch patches a generation row except in the mutation that ends its pass. - A reviewer confirms that the marker table's index leads with the tenant, that `applyAggregate` reads it before every aggregate write, and that it compares the marker's `sourceVersions` before adjusting the sum. - A reviewer confirms that no field, index or function of the registry carries a tenant, that every batch cursor names one, and that the restore's inventory reads `by_read_model_state` for the three in-flight states and nothing else. diff --git a/design/specs/application/projection-contract.sdp.md b/design/specs/application/projection-contract.sdp.md index 9f25693..6b4e3bb 100644 --- a/design/specs/application/projection-contract.sdp.md +++ b/design/specs/application/projection-contract.sdp.md @@ -30,33 +30,33 @@ A projection is the named, versioned, deterministic function that turns a contex ### Open questions -- [non-blocking] Extension E-9: the doc says a projection is named, versioned and deterministic and that live update and rebuild share its logic; it does not give the signature or the row shape, for counts and sums it names the marker but not the aggregate row, and for the read models that depend on history across several streams, the ones the write pause exists for, it names no input. The option taken here: a per-entity read model declared as a `ReadModel` with `name`, `table`, `rowBudgetBytes` and its `Projection`, which holds `version`, `keyOf` and `project` alone, rows with `tenantId`, `generation`, `key`, `projectionVersion` and `sourceVersions`, one `applyProjection` helper with a mode per caller that writes each generation by its role, active or building, for an `AggregateProjection` an aggregate row with `value` and `entityCount` and no `sourceVersions`, because the markers carry them, written by `applyAggregate` under the marker's version compare with the mode ignored, and for a history-dependent view a `HistoryProjection` that folds events into the prior row, fed on the live path by the events the operation appended, which the adapter's `streams` entry now carries, and on the backfill by the context's `history` pages under the write pause, with `sourceVersions` recording the highest version folded per stream so that an event is never folded twice; a cross-context or cross-stream row is built by one projection bound once per source stream type; the owner confirms the three forms and has not ruled on the split of a per-entity read model into `ReadModel` and `Projection`, on the write by generation role or on the row measured against its budget at every write; no check ties a generation to the projection version it records, and how a command writes two projection versions at once is the next question (E-9, E-23, E-24, D8, D9) -- [non-blocking] Extension E-9, two projection versions: a read model holds one projection, so `applyProjection` writes every generation it is given with that projection, whatever `projectionVersion` the generation records; after a projection version change the active generation was built by the earlier version, and the owner has not ruled how a command writes two projection versions at once, the active generation's and a building generation's (E-9, D8, D9) +- [non-blocking] Extension E-9: the doc says a projection is named, versioned and deterministic and that live update and rebuild share its logic; it does not give the signature or the row shape, for counts and sums it names the marker but not the aggregate row, and for the read models that depend on history across several streams, the ones the write pause exists for, it names no input. The option taken here: a per-entity read model declared as a `ReadModel` with `name`, `table`, `rowBudgetBytes` and its `projections`, a list of `Projection` with the latest first, each holding `version`, `keyOf` and `project` alone, rows with `tenantId`, `generation`, `key`, `projectionVersion` and `sourceVersions`, one `applyProjection` helper with a mode per caller that writes each generation by its role, active, building or verifying, with the projection whose version the generation records, for an `AggregateProjection` an aggregate row with `value` and `entityCount` and no `sourceVersions`, because the markers carry them, written by `applyAggregate` under the marker's version compare with the mode ignored, and for a history-dependent view a `HistoryProjection` that folds events into the prior row, fed on the live path by the events the operation appended, which the adapter's `streams` entry now carries, and on the backfill by the context's `history` pages under the write pause, with `sourceVersions` recording the highest version folded per stream so that an event is never folded twice; a cross-context or cross-stream row is built by one projection bound once per source stream type; the owner confirms the three forms and has not ruled on the split of a per-entity read model into `ReadModel` and `Projection`, on the write by generation role or on the row measured against its budget at every write (E-9, E-23, E-24, D8, D9) +- [non-blocking] Extension E-9, two projection versions: the doc says live update and rebuild share one projection and does not say how a command writes two generations built by two versions; the option taken here is that a read model declares its projection versions as a list, the latest first, and `applyProjection` writes each generation with the projection whose version the generation row records, failing as a technical failure when the read model declares none of that version, so the rows of one generation are one truth of one version; the cost is that an earlier version stays in the code while a generation that records it is written or can be rolled back to, and a projection changed without a version change is still undetected (E-9, D8, D9) - [non-blocking] The rule that a row with a newer version is never overwritten is read as the backfill path's rule only, so a live write to the active generation always lands, as `step3` has it; the owner has not confirmed that reading (D9, E-9) ## Contract - A projection has a version and a pure `project` function from a DTO at its stream version to a row or `null`, and is named by the read model it maintains; the same read model and version always give the same row for the same input (D8) - A projection reads only its input; it never queries the database, the network, the scheduler, the environment or `ctx.auth`, and it never issues commands or causes effects (D8, Law 10) -- Live update and rebuild call the same `applyProjection` with the same projection; only the mode differs (D8, D9) +- Live update and rebuild call the same `applyProjection` with the same read model, and each generation is written by the projection of the version it records; only the mode differs (D8, D9) - The pipeline's step 9 picks the live-created mode when the `created` flag on the operation outcome's `streams` entry for that stream is true, which the adapter sets from a load that found no row, and the live-updated mode otherwise; no use case derives the flag (D9, E-9, E-23) - The DTO a projection receives is the same `toDto(state, meta)` on both paths, the adapter's `streams` entry on the live path and the context's `list` page on the backfill, and it carries the stream's `StreamVersion` as `version`, so no adapter between two DTO shapes exists (D8, D9, E-9, E-24) - Every read-model row carries `tenantId`, `generation`, `key` and `projectionVersion`, a per-entity or cross-context row carries `sourceVersions` as well, and every read-model table has the `by_key` index leading with the tenant (E-9, Law 11, D9) - `sourceVersions` holds one `StreamVersion` per source stream the row derives from; a per-entity row holds one, a cross-context row holds one per context (E-9, D2, D8) - [extension] An aggregate row derives from every entity under its key, so it carries no `sourceVersions`; it carries `value`, the sum of the recorded contributions, and `entityCount`, the number of markers, and the per-entity marker carries the `sourceVersions` the row conventions would otherwise hold (E-9, D9) -- [extension] A read model declares its name, the parent table that holds its rows, `rowBudgetBytes`, the largest row it writes, 16 KiB by default and at most `limitRowSize`, and its projection; the projection is the versioned rules alone, so they are tested without a table; the rebuild derives its batch bound from the declared budget the way the context derives its page from the stream budget (E-9, E-40, F13) +- [extension] A read model declares its name, the parent table that holds its rows, `rowBudgetBytes`, the largest row it writes, 16 KiB by default and at most `limitRowSize`, and its projections, one per version it can still write, the latest first, with distinct versions and at least one; every version writes rows the one table's validator accepts, and a change of shape the table cannot hold beside the earlier one is a new read model; a projection is the versioned rules alone, so they are tested without a table; the rebuild derives its batch bound from the declared budget the way the context derives its page from the stream budget (E-9, E-40, F13) - On the backfill path a row is written only if it is missing or its `sourceVersions` are older than the input's for the same stream, and a row with a newer version is never overwritten; the rule is the backfill path's only, and a live write to the active generation always lands (D9) - In the active generation a live write always lands: a missing row is inserted and an existing row is replaced, in the live-created mode and the live-updated mode alike, because `project` sees the whole DTO and cannot build a partial row, so a subject whose row was deleted gets its row back from the next command that writes it (D9, E-9) -- In a building generation the live-created mode inserts or replaces the row, and the live-updated mode replaces a row the backfill has already written and skips a missing one (D9) +- In a generation in `building`, the role `building`, the live-created mode inserts or replaces the row, and the live-updated mode replaces a row the backfill has already written and skips a missing one; in a generation in `verifying` or `verified`, the role `verifying`, the backfill has ended, a missing row is a subject that projected to none, and a live write lands as it does in the active generation (D9, E-8, E-9) - [extension] Every write of a read-model row measures the whole row, the conventions included, with `getConvexSize` and throws a plain error, a technical failure that rolls the command back, when it measures above the read model's `rowBudgetBytes`; a declared budget above the ceiling of `limitRowSize` throws the same way (E-9, F13, Law 6) -- In the backfill mode the row is inserted or updated in the building generation only, if missing or older (D9) +- In the backfill mode the one generation the batch names is written whatever its role: the row is inserted if missing, updated if older, and deleted if present when the DTO projects to `null`, which is how a verify pass makes absence right as well as presence (D9, E-8, E-9) - A projection may return `null` for a DTO that has no row, such as a deleted subject; `applyProjection` then deletes the row if present and does nothing when it is absent, and nothing records the deletion, because a backfill batch reads the source in the same mutation that writes the row, so a batch that read the subject before its deletion committed re-runs under optimistic concurrency and reads the deleted subject, as the rebuild's `occRace` states (E-9, D2, F1) - An aggregate projection contributes a number per entity to an aggregate row and keeps a per-entity marker so live commands and backfill never count one entity twice (D9) - [extension] `applyAggregate` ignores the mode: the marker's `sourceVersions` decide, a marker that is not older than the input makes the write `skipped-newer`, an older or missing marker is written with the new contribution and versions and the difference goes to `value`, and a `null` contribution removes the marker and its recorded contribution; a live update on an entity backfill has not reached is therefore not partial, because `contribute` sees the whole DTO, which is why the missing-row rule of the per-entity path does not apply (E-9, D9, F1) - [extension] A history projection folds events into a row: `keyOf` maps each event to the row it belongs to or to none, and `project` folds the events of one key, in stream-version order within each stream, into the prior row; it is deterministic in the prior row and the events, reads nothing else, and is the one form a read model that depends on history across several streams may take, which is why such a read model rebuilds only under the write pause (E-9, D8, D9, Law 10) - [extension] A history row's `sourceVersions` hold, per source stream, the highest event version folded into it; `applyHistoryProjection` drops every event whose version is not above the recorded version for its stream and skips the write when none remain, so a live update and a backfill page never fold one event twice, and the live path never meets a non-active generation of a history view because its building generation exists only while its source scopes are closed (E-9, D9, F1) - [extension] The same projection is bound once per source stream type on the command declaration; the pipeline's step 9 feeds it each matching entry's events, and the backfill feeds it each source's history, so a cross-stream row has one projection and several bindings (E-9, E-7, D8) -- A projection's version changes whenever its output for any input changes; a new version is a new generation, never an in-place rewrite (D8, D9) +- A projection's version changes whenever its output for any input changes; a new version is added at the head of the read model's `projections` and is a new generation, never an in-place rewrite, and an earlier version leaves the list only when no generation in `building`, `verifying`, `verified` or `active` records it and none in `retired` is to be rolled back to (D8, D9) - The write rule reads and writes inside one mutation, so two writers of the same row serialize under optimistic concurrency and neither overwrites newer data (F1, D9) ## Design @@ -64,8 +64,9 @@ A projection is the named, versioned, deterministic function that turns a contex The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape with its `version` field, and `streamVersionValidator` is the kernel's validator; `D` is the context's DTO type; `Row` is the read-model document without the row conventions, which `applyProjection` adds. Validators use `convex/values`; the row conventions are a validator the read-model tables spread into their `defineTable`. - typeProjection: `type Projection> = { version: number; keyOf(tenantId: string, dto: D): string; project(tenantId: string, dto: D, versions: StreamVersion[]): Row | null }` (E-9, D8) -- typeReadModel: `type ReadModel> = { name: string; table: string; rowBudgetBytes: number; projection: Projection }` where `name` is the read model's key in the generation registry, `table` is the parent table that holds its rows, and `rowBudgetBytes` is at most 65,536 with `defaultRowBudgetBytes` 16,384 (E-9, D8, F13) -- typeWritableGeneration: `type WritableGeneration = { role: "active" | "building"; generation: number; projectionVersion: number }`, one generation a command writes, as the registry's `generationsToWrite` returns it (E-8, E-9, D9) +- typeReadModel: `type ReadModel> = { name: string; table: string; rowBudgetBytes: number; projections: readonly [Projection, ...Projection[]] }` where `name` is the read model's key in the generation registry, `table` is the parent table that holds its rows, `rowBudgetBytes` is at most 65,536 with `defaultRowBudgetBytes` 16,384, and `projections` lists the versions the deployment can write, the latest first (E-9, D8, F13) +- fnProjectionOf: `projectionOf>(readModel: ReadModel, version: number): Projection` returning the declared projection of that version and throwing the plain `Error` `"Read model " + readModel.name + " declares no projection of version " + version` when there is none, a technical failure on the command path and a refusal of `startGeneration` and `rollbackGeneration` (E-9, D8, Law 6) +- typeWritableGeneration: `type WritableGeneration = { role: "active" | "building" | "verifying"; generation: number; projectionVersion: number }`, one generation a command writes, as the registry's `generationsToWrite` returns it; the role `verifying` covers the states `verifying` and `verified` (E-8, E-9, D9) - typeRowWriter: `type RowWriter = { db: GenericDatabaseWriter }`, the part of a ctx `applyProjection` uses, generic over the composition's data model because a database writer is not covariant in its data model; `RowDataModel` is the data model of one table that spreads `rowConventions` and has `by_key`, and the library reads and writes the read model's own table, named by its `table`, under that table's type (E-9) - typeAggregateProjection: `interface AggregateProjection { name: string; version: number; table: string; rowBudgetBytes?: number; aggregateKeyOf(tenantId: string, dto: D): string; contribute(tenantId: string, dto: D): number | null }` where `null` means the entity no longer contributes, such as a deleted subject, and `rowBudgetBytes` bounds the aggregate row and the marker alike (E-9, D9, F13) - typeHistoryProjection: `interface HistoryProjection { name: string; version: number; table: string; rowBudgetBytes?: number; keyOf(tenantId: string, event: EventEnvelope): string | null; project(tenantId: string, prior: Row | null, events: EventEnvelope[]): Row | null }` where `EventEnvelope` is the envelope of `spec:context.event-envelope`, `null` from `keyOf` means the event is not this view's, `events` are the events of one key not yet folded into `prior`, in version order within each stream, and `null` from `project` deletes the row; a history row is a bounded summary of history, never a list that grows with it, so `rowBudgetBytes` binds it like any row (E-9, D8, D9, F13) @@ -73,7 +74,7 @@ The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape w - typeApplyMode: `type ApplyMode = "live-created" | "live-updated" | "backfill"` (E-9, D9) - typeApplyResult: `type ApplyResult = "inserted" | "updated" | "deleted" | "skipped-missing" | "skipped-newer" | "unchanged"` (E-9, D9) - fnApplyProjection: `applyProjection>(ctx: RowWriter, readModel: ReadModel, input: { tenantId: string; dto: D; versions: StreamVersion[]; mode: ApplyMode; generations: readonly WritableGeneration[] }): Promise` writing one result per generation in the order given; the pipeline's step 9 calls it with `versions: [entry.version]` for a per-entity row and the rebuild's batch with `versions: [dto.version]` (E-9, D8, D9) -- callers: the two callers are the command pipeline's step 9, with the mode from the `streams` entry's `created` flag and the list `generationsToWrite` returned, passed unchanged, and the rebuild's backfill and verify batches in `backfill` mode for the building generation; a use case body never calls it, and the same two callers are the only callers of `applyAggregate` and `applyHistoryProjection` (E-9, D8, D9, E-7) +- callers: the two callers are the command pipeline's step 9, with the mode from the `streams` entry's `created` flag and the list `generationsToWrite` returned, passed unchanged, and the rebuild's backfill and verify batches in `backfill` mode for the one generation they build or verify; a use case body never calls it, and the same two callers are the only callers of `applyAggregate` and `applyHistoryProjection` (E-9, D8, D9, E-7) - tableOrderAllocationHistory: `orderAllocationHistory: defineTable({ ...rowConventions, orderId: v.string(), lines: v.array(v.object({ stockItemId: v.string(), quantity: v.number(), allocatedAt: v.optional(v.number()), releasedAt: v.optional(v.number()) })) }).index("by_key", ["tenantId", "generation", "key"])` the history-dependent cross-stream view of the example domain, keyed by order ID and folded from `OrderPlaced` in Orders and `StockAllocated` and `AllocationReleased` in Inventory, whose row is bounded by the order's line count and whose `sourceVersions` hold the order stream and every stock stream folded; it is bound once per source stream type and rebuilds only under the write pause, and Sc L2-6 builds it (E-9, E-46, D9, Sc L2-6) - fnApplyAggregate: `applyAggregate(ctx: MutationCtx, projection: AggregateProjection, input: { tenantId: string; entityKey: string; dto: D; versions: StreamVersion[]; mode: ApplyMode; generations: readonly WritableGeneration[] }): Promise` writing one result per generation, `inserted` when the aggregate row was created, `updated` when `value` or `entityCount` changed, `skipped-newer` when the marker was not older, and `unchanged` when nothing moved; `mode` is accepted for symmetry with `applyProjection` and read by nothing (E-9, D9) - aggregateStep1: for each generation it reads the marker by `by_entity` with the tenant, read model, generation and `entityKey`, and computes `contribution` as `contribute(tenantId, dto)` (E-9, D9) @@ -86,10 +87,10 @@ The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape w - tableOrderSummaries: `orderSummaries: defineTable({ ...rowConventions, ...orderSummaryFields }).index("by_key", ["tenantId", "generation", "key"]).index("by_status", ["tenantId", "generation", "status", "placedAt"])` (E-9, E-46, First experiment) - indexByKeyUse: point read by `applyProjection` before every write and by the entity query; `key` is `keyOf(tenantId, dto)`, the application ID of the entity (E-9, D2) - indexByStatusUse: the essential list query of the first experiment, paginated, within the active generation (E-9, First experiment) -- step1: `applyProjection` computes `key` and the candidate row with `project`; a `null` candidate means delete the row if present and `unchanged` when there is none (E-9) -- step2: for each generation in `generations` it reads the row by `by_key` (E-9, D9) -- step3: it decides by the generation's role and the mode: in the active generation it inserts a missing row and replaces an existing one in both live modes; in a building generation live-created inserts or replaces, live-updated replaces an existing row and answers `skipped-missing` for a missing one, and backfill inserts when missing and updates when the existing `sourceVersions` are older, else `skipped-newer` (D9, E-9) -- step4: before a write it throws a plain error when the row measures above `rowBudgetBytes` with `getConvexSize`; then it writes with `ctx.db.insert`, `ctx.db.replace` or `ctx.db.delete` in the same mutation, a written row carrying the declared projection's version as `projectionVersion`, and returns the results in generation order (E-9, F1, F13) +- step1: for each generation in `generations`, `applyProjection` takes `projectionOf(readModel, generation.projectionVersion)` and computes `key` and the candidate row with that projection's `keyOf` and `project`; a `null` candidate means delete the row if present, answering `deleted`, and `unchanged` when there is none, in every mode and role (E-9, D8) +- step2: it reads that generation's row by `by_key` (E-9, D9) +- step3: it decides by the mode and the generation's role: in the two live modes it inserts a missing row and replaces an existing one, except that live-updated in the role `building` answers `skipped-missing` for a missing row; in the backfill mode it inserts when missing and updates when the existing `sourceVersions` are older, else `skipped-newer`, whatever the role (D9, E-8, E-9) +- step4: before a write it throws a plain error when the row measures above `rowBudgetBytes` with `getConvexSize`; then it writes with `ctx.db.insert`, `ctx.db.replace` or `ctx.db.delete` in the same mutation, a written row carrying the generation's `projectionVersion`, which is the version of the projection that built it, and returns the results in generation order (E-9, F1, F13) - limitRowSize: a read-model row stays under its read model's `rowBudgetBytes`, which `applyProjection` checks at every write, 16 KiB by default and at most 64 KiB, the library constant `limitRowBudgetBytes`, well under the 1 MiB document limit; the rebuild's batch bound is derived from the declared budget, and a row that would grow without bound with history is a wrong read model for this contract, a history projection included, whose row summarizes history within the same budget (F13, E-9, E-40) - limitHistoryFold: on the live path `applyHistoryProjection` reads and writes at most one row per distinct key among one entry's events, each at most `rowBudgetBytes`, so a command that appends `n` events touches at most `n` history rows; on the backfill it takes one `history` page bounded by the rebuild's `limitHistoryBatch`, so one batch reads at most that many events plus as many rows (F13, E-9, E-40) - limitGenerationsPerWrite: at most 2 generations are written per row per command, the active one and the one in building, verifying or verified if any; a retired generation is kept but never written, so one read model costs at most 2 point reads and 2 writes per command and exactly 1 of each outside a rebuild (E-9, E-8, F13) @@ -102,3 +103,4 @@ The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape w - A reviewer confirms that no `project` function touches `ctx`, the database, the scheduler or the environment. - A reviewer confirms that every per-entity or cross-context read-model table spreads `rowConventions`, that every aggregate table spreads `aggregateRowConventions`, and that each carries the `by_key` index. - A reviewer confirms that `applyAggregate` compares the marker's `sourceVersions` before it touches `value`, that it reads no mode, and that a `null` contribution removes the marker and its recorded contribution. +- A reviewer confirms that a test writes one command into an active generation of one projection version and a generation being built of another and finds on each row its own generation's `projectionVersion` and that version's output, and that a test deletes a row through a `null` projection and finds the next live-updated write answering `inserted` in the active generation and in a generation in `verifying`, and `skipped-missing` in one in `building`. diff --git a/design/specs/application/read-models.committed-state-read-through-query.sdp.md b/design/specs/application/read-models.committed-state-read-through-query.sdp.md index 439cb46..f765828 100644 --- a/design/specs/application/read-models.committed-state-read-through-query.sdp.md +++ b/design/specs/application/read-models.committed-state-read-through-query.sdp.md @@ -25,5 +25,5 @@ And the number of workers, jobs or queues that ran is {workers: 0} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- After grants, the first activation and `ReceiveStock` as setup, the test runs `PlaceOrder` and then, from the same client, calls the parent query `getOrder`, which relays the Orders context's `get`, and asserts that the DTO it returns holds the order's committed state and that its `version` equals the command's returned `versions` entry for the order's stream. +- After grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test runs `PlaceOrder` and then, from the same client, calls the parent query `getOrder`, which relays the Orders context's `get`, and asserts that the DTO it returns holds the order's committed state and that its `version` equals the command's returned `versions` entry for the order's stream. - The test asserts that no scheduled function, cron or action ran for the command by reading the disposable backend's function log from a mark before the command to the `getOrder` read that the same client sends after the command returned, a window an ordinary read closes, and finding no record that a client did not cause; a job scheduled with a delay past that window is not seen. diff --git a/design/specs/application/read-models.committed-state-visible.sdp.md b/design/specs/application/read-models.committed-state-visible.sdp.md index a0736a3..7ec446a 100644 --- a/design/specs/application/read-models.committed-state-visible.sdp.md +++ b/design/specs/application/read-models.committed-state-visible.sdp.md @@ -26,5 +26,5 @@ And the number of workers, jobs or queues that ran is {workers: 0} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- After grants, the first activation and `ReceiveStock` as setup, the test subscribes to `listOrderSummaries` with the tenant, the status a placed order has and `{ cursor: null, numItems: 10 }`, runs `PlaceOrder`, and asserts that the next subscription value's page carries the order's summary whose `sourceVersions` equal the command's returned `versions` entry for the order's stream, the one stream the summary is projected from. +- After grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test subscribes to `listOrderSummaries` with the tenant, the status a placed order has and `{ cursor: null, numItems: 10 }`, runs `PlaceOrder`, and asserts that the next subscription value's page carries the order's summary whose `sourceVersions` equal the command's returned `versions` entry for the order's stream, the one stream the summary is projected from. - The test asserts that the backend ran no scheduled function, cron or action between the command and the subscription update, by reading the disposable backend's function log from before the command until it holds the command's mutation and the subscription's query, a window an ordinary read after the command closes, and finding no record that a client did not cause; a job scheduled with a delay past that window is not seen. diff --git a/design/specs/application/read-models.list-pages-by-cursor.sdp.md b/design/specs/application/read-models.list-pages-by-cursor.sdp.md index d126929..9c206fc 100644 --- a/design/specs/application/read-models.list-pages-by-cursor.sdp.md +++ b/design/specs/application/read-models.list-pages-by-cursor.sdp.md @@ -26,5 +26,5 @@ And the number of pages that carry pageStatus SplitRequired is {splitPages: 0} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- After grants, the first activation and `ReceiveStock` as setup, the test places the orders through `PlaceOrder`, then reads `listOrders` for the first tenant as a caller granted `orders.read`, each page with `cursor` and `numItems` and then subscribed with `endCursor` set to the `continueCursor` that read returned, the next page starting at the previous page's `endCursor`, until a read answers `isDone`; with every page subscribed, it places two orders whose order IDs sort inside the first page, `order-00a` and `order-05a`. +- After grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test places the orders through `PlaceOrder`, then reads `listOrders` for the first tenant as a caller granted `orders.read`, each page with `cursor` and `numItems` and then subscribed with `endCursor` set to the `continueCursor` that read returned, the next page starting at the previous page's `endCursor`, until a read answers `isDone`; with every page subscribed, it places two orders whose order IDs sort inside the first page, `order-00a` and `order-05a`. - The test asserts that the first page, pinned by its end cursor, still starts and ends at the orders it held and grows from 10 to 12 orders, that the second page starts at the order after it, that the pages then hold 12, 10 and 5 orders, that no page carries `pageStatus` `SplitRequired`, and that every order ID of the first tenant appears once, none missing or repeated. diff --git a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md index bb9fcfd..4d8640f 100644 --- a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md +++ b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md @@ -30,7 +30,9 @@ And writes {writes: "never stopped"} ## Verification — executable - Runs in the native tier; every test owns its disposable backend. -- The test seeds 350 orders across two tenants, 250 in the first and 100 in the second, starts a generation of the order summary with the default batch size, which the order stream's default 256 KiB budget caps at 16 items per `list` page, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and kills the backend's scheduler after the third batch, so at most 48 orders of the first tenant are backfilled and the second tenant is untouched. +- The test seeds 350 orders across two tenants, 250 in the first and 100 in the second, with the order summary's generation 1 active, starts generation 2 with a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that `batchesDone` then stops advancing below 350 and that the cursor names a tenant. - The test asserts that a command on an order whose row backfill had not yet written left no row in the building generation, and that a command on an order whose row existed updated it. -- After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that every order enumerated by the Orders context in either tenant has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, and that the backend's function log shows no context operation call and no action during the batches. -- The test switches, asserts that the subscriptions on the list query in both tenants re-ran with the new generation, runs one more `PlaceOrder`, rolls back, asserts that the reopened generation reaches `verified` with a row for the new order before it becomes active, and then asserts that the old generation is active and the subscriptions re-ran with it. +- After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that, once the generation is `verified`, every order enumerated by the Orders context in either tenant has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, that a batch scheduled under the fence the interrupt replaced wrote nothing, and that the backend's function log shows no context operation call and no action during the batches. +- The test asserts that no command of the run failed for reading the registry while the batches wrote their checkpoints, and that the generation rows changed only at the start, at the end of the backfill, at the end of the verify pass and at each operator entry. +- The test switches, asserts that the subscriptions on the list query in both tenants re-ran with the new generation and that generation 1 is `retired`, then runs one more `PlaceOrder` and deletes or cancels one earlier order so that its summary changes, rolls back by naming generation 1, asserts that the reopened generation reaches `verified` after one verify pass with a row for the new order and the changed order's current row before it becomes active, switches again, and asserts that generation 1 is active, generation 2 is `retired` and the subscriptions re-ran. +- A `convex-test` test of the same workflow runs each batch by calling it directly, so it interrupts after exactly 3 batches, rebuilds with a second projection version and asserts that each generation's rows carry their own generation's `projectionVersion` and that version's output under live writes, deletes a subject between the switch and the rollback and asserts that the reopened generation holds no row for it after its verify pass, and asserts every refusal of the operator entries by its message. diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 9d17e77..1bea44d 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -32,100 +32,118 @@ relations: --- # Rebuild: generations, backfill and the write pause -Layer 2 · Detail: full · Traces: D8, D9, D11, D19, Law 10, Law 11, F1, F7, F8, F9, F12, F13, F17, Probe 6, Sc L2-5, Sc L2-6, E-8, E-40, E-42. +Layer 2 · Detail: full · Traces: D8, D9, D11, D19, Law 10, Law 11, F1, F7, F8, F9, F12, F13, F17, Probe 6, Sc L2-5, Sc L2-6, E-8, E-9, E-40, E-42, E-48, E-49. -A rebuild builds a new numbered generation of a read model while the old one stays active. For a per-entity or current-state read model it runs online: the generation is registered as building, live commands write it as well as the active one, backfill batches fill it from each source at its current stream version writing only where the target is missing or older, and one write switches the active generation after coverage is verified. Optimistic concurrency orders each batch against live commands, so neither side overwrites newer data. The old generation is kept for a rollback period and no longer written; a rollback reopens it and one verify pass repairs it before it is active again. For a read model that depends on history across several streams the rebuild runs under a write pause, because it needs a consistent cut; the scope closed is the view's sources, the context and stream type pairs its bindings name, so every use case that writes one of them and the journal's baseline driver read the gate and are refused, background writers are fenced by that read and by the generation's fence, and an operator can resume or abort. Such a view is a `HistoryProjection` and its backfill pages each source stream's history under the pause. Rebuild never runs commands or external effects. +A rebuild builds a new numbered generation of a read model while the old one stays active, and it is also how a read model comes to exist: its first generation is built the same way. For a per-entity read model it runs online: the generation is registered as building, live commands write it as well as the active one, backfill batches fill it from each source at its current stream version writing only where the target is missing or older, one verify pass makes presence and absence right, and one write switches the active generation. Optimistic concurrency orders each batch against live commands, so neither side overwrites newer data. The old generation is kept for a rollback period and no longer written; a rollback reopens it, one verify pass repairs it, and the same switch makes it active again. For a read model that depends on history across several streams the rebuild runs under a write pause, because it needs a consistent cut; the scope closed is the view's sources, the context and stream type pairs its bindings name, so every use case that writes one of them and the journal's baseline driver read the gate and are refused, background writers are fenced by that read and by the generation's fence, and an operator can resume or abort. Such a view is a `HistoryProjection` and its backfill pages each source stream's history under the pause. Rebuild never runs commands or external effects. -The registry, the per-entity marker and the gate document are pinned by `spec:application.generation-registry` and `spec:application.write-pause`. The projection and the write rule are `spec:application.projection-contract`. Sources are read through the context's queries, never its tables; a rebuild of a stream from its own events is the journal's concern and not this one. +The registry, the progress row, the per-entity marker and the gate document are pinned by `spec:application.generation-registry` and `spec:application.write-pause`. The projection and the write rule are `spec:application.projection-contract`. The operator entry and the tenant list are `spec:command.actor-and-scope`. Sources are read through the context's queries, never its tables; a rebuild of a stream from its own events is the journal's concern and not this one. ## Intent -- actor: The operator who starts, verifies, switches, rolls back or aborts a generation, and the use cases whose live writes keep the building generation current (D9, D19) -- problem: A per-entity read model is rebuilt while commands run and the rebuild is interrupted and resumed; a live command's incremental update on a row backfill has not written builds a partial one, a backfill batch overwrites newer data, an entity is missed at cutover, or a side effect replays; a cross-stream history view is rebuilt without a consistent cut (Sc L2-5, Sc L2-6) -- outcome: A per-entity or current-state read model rebuilds online as a new generation with no stale overwrite, no missing entity and no side effect, with cutover and rollback in one write each; a history-dependent view rebuilds under a write pause with safe resume and abort, and writes reopen only after verification (D9, Sc L2-5, Sc L2-6) +- actor: The operator who starts, interrupts, resumes, switches, rolls back, aborts or purges a generation through the operator entries, and the use cases whose live writes keep the generation being built current (D9, D19) +- problem: A per-entity read model is rebuilt while commands run and the rebuild is interrupted and resumed; a live command's incremental update on a row backfill has not written builds a partial one, a backfill batch overwrites newer data, an entity is missed at cutover, a deleted subject keeps its row, or a side effect replays; a cross-stream history view is rebuilt without a consistent cut (Sc L2-5, Sc L2-6) +- outcome: A per-entity read model rebuilds online as a new generation with no stale overwrite, no missing entity, no row of a subject that projects to none and no side effect, with the cutover in one write and a rollback that is one verify pass and then that switch of one write; a history-dependent view rebuilds under a write pause with safe resume and abort, and writes reopen only after verification (D9, Sc L2-5, Sc L2-6) - value: The common rebuild costs no downtime and no projection job; the write pause is paid only where a consistent cut is needed (D9) -- risk: The standing cost is a generation registry read in every command that maintains a read model, a generation on every row, a marker per entity for counts and sums, the batch runner, and for the write pause a gate every writer reads (D9) -- risk: The batch bound and the runner are design choices under the transaction limits until Probe 6 shows how a batch races a live command and whether the migrations component fits (F13, F17, Probe 6) -- assumption: Mutations are serializable under optimistic concurrency, so a backfill batch and a live command that touch the same row serialize and the loser retries (F1) -- assumption: `@convex-dev/migrations` fits a generation backfill; its product page states resumable batches with saved progress, resume from the last checkpoint and running over a subset, and states no batch size, so the batch bound below is this design's own under E-40 and F13, and whether a migration can iterate a table inside a component is what Probe 6 shows (F17, Probe 6) +- risk: The standing cost is a generation registry read in every command that maintains a read model, a generation on every row, the earlier projection version kept in the code while its generation can be rolled back to, the batch chain, and for the write pause a gate every writer reads (D9) +- risk: A pass costs at least one batch per tenant of the tenant list, because a batch names one tenant, so a deployment of many empty tenants pays one scheduled mutation for each (E-49, Law 11) +- risk: The batch bound is a design choice under the transaction limits until Probe 6 shows how a batch races a live command (F13, Probe 6) +- assumption: Mutations are serializable under optimistic concurrency, so a backfill batch and a live command that touch the same row serialize and the loser re-runs (F1) +- assumption: A scheduled mutation that loses an optimistic-concurrency race is run again by Convex until it commits; the pinned packages do not show a bound on those runs, the design needs none for correctness, and Probe 6 shows what a stalled batch looks like (F9, F1, Probe 6) - assumption: A parent query calling a component query stays reactive, so switching the active generation in the registry re-runs every subscription that reads it, and the enumeration's `paginator` page relayed by the parent stays contiguous under live writes; this is F15, and Probe 5 showed both on the pinned backend (F15, Probe 5, S4) - assumption: Scheduling from a mutation commits with it, and a scheduled mutation retries internal errors and runs once (F8, F9) -- assumption: Backups exclude pending scheduled functions, so a restore drops the batch that was driving a generation and leaves its row at the checkpoint; the restore procedure's inventory and resume are the recovery (F12, E-42) +- assumption: Backups exclude pending scheduled functions, so a restore drops the batch that was driving a generation and leaves its rows at the checkpoint; the restore procedure's inventory and resume are the recovery (F12, E-42) ### Open questions -- [non-blocking] Probe 6 pending: a backfill batch racing a live command under optimistic concurrency, and whether the migrations component can drive a batch whose source is a context's enumeration rather than a parent table (Probe 6, F1, F17, D9) -- [non-blocking] Extension E-40: the doc binds resumable batching to `@convex-dev/migrations` and gives no batch size; the component iterates a parent table while the sources of a read model live in a context component. The option taken here: a batch is one mutation with one cursor and a bound of at most 100 entities, capped twice, by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget, a design choice under F13 that is a field of the generation row and that `resumeGeneration` lowers down to 1 when a generation stalls on optimistic-concurrency conflicts over a hot key range, because a mutation that loses the race never commits and cannot count its own losses; the migrations component drives batches whose source is a parent table, and a self-scheduled internal mutation with the same batch contract drives batches whose source is a context's enumeration until Probe 6 shows the component can; Probe 6 also measures the contention and decides whether an automatic stall check earns its function (E-40, E-24, E-9, F17, F13, F1) -- [non-blocking] Extension E-8: the registry states, the rollback period, what a rollback does, the gate document, the scope a paused rebuild closes and the reach of a generation are design choices the doc does not make; they are pinned in the two contracts under this workflow, the registry's E-8 question records the two readings of rollback, kept-and-reverified against write-through, with the cheaper one taken, and it records the two readings of reach, one generation per read model spanning every tenant against one registry per read model and tenant, with the deployment-wide one taken because a generation is a build of the projection's code, which is the same for every tenant; the write pause's E-8 question records that a paused rebuild closes the view's source stream types rather than the read model, because a consistent cut must hold every writer of the sources, which the declaration's `writes` makes derivable; and this workflow adds that the backfill of a history view pages each source stream's history one page per batch, with the stream and the event cursor on the batch cursor, because a `HistoryProjection` takes events and the doc's write-pause rebuild has no other input (E-8, E-9, E-38, D9) +- [non-blocking] Probe 6 pending: a backfill batch racing a live command under optimistic concurrency, and whether `@convex-dev/migrations` can drive a batch from inside or across a component boundary (Probe 6, F1, F17, D9) +- [non-blocking] Extension E-40: the doc binds resumable batching to `@convex-dev/migrations` and gives no batch size; the component iterates a parent table while the sources of a read model live in a context component. The option taken here: a batch is one mutation with one cursor on the progress row and a bound of at most 100 entities, capped by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget; a batch whose source is a context's enumeration, the backfill and the verify pass, is driven by a self-scheduled internal mutation, and a batch whose source is a parent table, the purge, is the component's to drive under the same batch contract once Probe 6 shows it fits; until then the purge is self-scheduled too, `purgeDriver` is the one sentence the probe changes, and the progress row stays the one checkpoint either way (E-40, E-24, E-9, F17, F13, F1) +- [non-blocking] Extension E-8: the registry states, the rollback period, what a rollback does, the gate document, the scope a paused rebuild closes and the reach of a generation are design choices the doc does not make; they are pinned in the two contracts under this workflow (E-8, E-9, E-38, D9) +- [non-blocking] Extension E-8, what this workflow does not offer: the online path is pinned for a per-entity read model with one source stream type; an `AggregateProjection` and a row built from more than one source stream type have no rebuild here, and each gets one when a view needs it; the paused path of a history view is given below in outline, its cursor across several sources and its verify pass are that view's own rules, and `rollbackGeneration` refuses a generation that needs a write pause, because nothing would hold its sources while live commands fold onto a row that missed the events since its retirement (E-8, E-9, D9, D18) ## Workflow - rule: Rebuild never runs commands or external effects (D9, Law 10) - rule: Rebuild reads a source only through the context's queries, with the tenant as the one argument that scopes them, never through the context's tables (D10, D11, F11) - rule: Online rebuild is the default for per-entity and current-state read models; the write pause is the exception, for read models that depend on history across several streams (D9) -- rule: Live commands also write the building generation; a command that creates a stream writes its row, and a command on an existing stream updates the row only if backfill has already written it (D9) -- rule: A backfill batch reads or folds each source at its current stream version and writes the row only if the target is missing or older (D9) -- rule: Coverage is verified before the switch, the active generation switches in one write, and the old generation is kept for a rollback period (D9) -- rule: Counts and sums use a per-entity marker so live commands and backfill never count one entity twice, which is still transactional (D9) +- rule: Live commands also write the generation being built; a command that creates a stream writes its row, and while the generation is `building` a command on an existing stream updates the row only if backfill has already written it (D9) +- rule: [extension] Once the backfill has ended, in `verifying` and `verified`, a live write always lands as it does in the active generation, because a missing row then means a subject that projected to none and never a row backfill has yet to write; this is what lets one verify pass be enough (E-8, E-9, D9) +- rule: A backfill batch reads each source at its current stream version and writes the row only if the target is missing or older (D9) +- rule: [extension] Coverage is verified before the switch by one verify pass over every tenant and every source, which inserts a missing row, updates an older one and deletes the row of a subject whose DTO projects to `null`; the generation is `verified` when that pass ends, whatever it repaired, and the count of repairs is kept for the operator (E-8, D9, Sc L2-5) +- rule: [extension] The active generation switches in one write: one mutation that patches two generation rows together, the verified one to `active` and the active one to `retired`, or one row when the read model had no active generation, so no command and no query sees one row changed without the other; the old generation is kept for a rollback period (D9, E-8, F1) +- rule: [extension] A retired generation is not written; a rollback reopens the retired generation the operator names as `verifying`, one verify pass repairs what the writes since its retirement changed, presence and absence alike, and the ordinary switch makes it active, so a rollback is a verify pass and then a switch of one write, never one write by itself, and the current generation serves until that switch (E-8, D9, Sc L2-5) +- rule: [extension] Each generation is written with the projection whose version its row records, by its batches and by live commands alike, and `startGeneration` and `rollbackGeneration` refuse a version the read model does not declare (E-9, E-8, D8) - rule: Under the write pause every writer in the scope reads the maintenance gate, background writers are fenced or drained, and an operator can resume or abort (D9) - rule: A refused write under the pause is a transient refusal, retryable and never a stored rejection (D4, D9) - rule: Every batch has a stated bound below the transaction limits, tested against the pinned Convex version; the limits are ceilings, not batch sizes (D19, F13) -- rule: [extension] The rebuild runs under an operator actor, and a generation spans every tenant of the deployment: its batches iterate the application's tenant list and name the tenant on every context call, so no call runs with an absent tenant as a wildcard, and the one registry row per read model answers every tenant's queries, which is why a switch can never leave one tenant reading a generation that was built for another (D11, Law 11, E-8) +- rule: [extension] Every function that starts or changes a generation is an operator entry of `spec:command.actor-and-scope`: an internal function that records the stated operator on the generation row and reads no grant (E-48, D11, D19) +- rule: [extension] A generation spans every tenant of the deployment: its batches take one tenant at a time from `nextTenant` of the tenant list, in tenant-ID order, and name that tenant on every context call, so no call runs with an absent tenant as a wildcard, and the one registry row per read model answers every tenant's queries (D11, Law 11, E-8, E-49) +- rule: [extension] A tenant whose row enters the tenant list behind a backfill's cursor is not visited by that pass and needs no visit, because every stream it owns was created after the generation was registered and so was written live; the verify pass starts again at the first tenant and visits it (E-49, E-8, D9) - rule: A backfill batch racing a live command is ordered by optimistic concurrency; the batch's read-then-write of a row is one mutation, so the batch never overwrites a row a command wrote after the batch read its source (D9, F1) -- rule: [extension] The batch runner is `@convex-dev/migrations` where the batch's source is a parent table, and a self-scheduled internal mutation with the same batch contract where the source is a context's enumeration (E-40, F17, F8) +- rule: [extension] A batch whose source is a context's enumeration is driven by a self-scheduled internal mutation, and a batch whose source is a parent table by `@convex-dev/migrations` under the same batch contract where Probe 6 shows the component fits (E-40, F17, F8) - rule: [extension] A projection version change is a new generation, never an in-place rewrite of the active one (E-8, D8) -- rule: [extension] The batch bound is a field of the generation row, at most 100 entities, at most `floor(4 MiB / (2 * rowBudgetBytes))` for the read model's declared row budget, and never more than the context's `list` page admits at the stream type's document budget; only `resumeGeneration` changes it, lowering it down to 1 when a generation stalls on optimistic-concurrency conflicts; a batch that loses the race never commits, so it cannot record its own loss (E-40, E-24, E-9, F1, F13) -- An operator calls `startGeneration` for a read model; it inserts the generation row in state `building` with the next generation number, the projection version, the batch size and fence 1, and schedules the first backfill batch in the same transaction; with `pauseRequired` it also closes the maintenance gate for every source scope of the read model's bindings in that transaction, before the batch is scheduled (D9, F8, E-8, E-38) -- From that commit every use case that maintains the read model writes the active generation and the building one through `applyProjection`, in live-created mode for a stream it created and live-updated mode otherwise (D9, E-9) -- Each backfill batch runs as one mutation: it loads the generation row, stops if the state is not `building` or the fence differs, reads the gate, calls the context's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the generation row, and schedules the next batch or moves the state to `verifying` when the enumeration is done (D9, D10, F1, E-40) -- A batch that is interrupted, times out or throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, a batch that keeps losing the optimistic-concurrency race fails after Convex's own retries with its documents-changed error, and a developer error, that failure or a timeout ends the batch, after which the operator's `resumeGeneration` bumps the fence, lowers the batch size when the failure was contention, and schedules a batch from the saved cursor; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, because the snapshot excludes scheduled functions while it brings the row back at its checkpoint, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) -- Each verify batch enumerates the sources again from the verify cursor and checks that every stream whose DTO projects to a row has one whose versions are at least the enumerated versions, while a stream whose DTO projects to `null` needs no row; a miss is backfilled in place and counted; the state moves to `verified` when the verify cursor is done and the last full pass counted zero misses (D9, Sc L2-5) -- `switchGeneration` sets the verified generation `active` and the active one `retired` with `retireAfter` in one write; every query reads the active generation from the registry, so every subscription on the read model re-runs (D9, F7) -- [extension] During the rollback period the retired generation's rows are kept and no longer written; `rollbackGeneration` reopens the retired generation as `verifying` with a bumped fence and a fresh verify cursor, live commands write it again from that commit, its verify pass backfills every row that is missing or older, and the ordinary `switchGeneration` makes it active again, so the current generation keeps serving until the rollback is verified (E-8, D9, Sc L2-5) -- After `retireAfter`, `purgeGenerationBatch` deletes the retired generation's rows and markers in bounded batches and marks it `purged` (D19, E-40) +- rule: [extension] The batch bound is a field of the progress row, a whole number of at least 1 and at most `rebuildBatchCeiling`, and never more than the context's `list` page admits at the stream type's document budget; only `resumeGeneration` changes it (E-40, E-24, E-9, F1, F13) +- rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them returns without writing (E-40, E-8, F8) +- An operator calls `startGeneration` for a read model; it inserts the generation row in state `building` with the next generation number, the projection version and fence 1, inserts the progress row in the phase `backfill` with the batch size and a `null` cursor, and schedules the first backfill batch, all in one mutation; a read model with no generation gets generation 1 this way (D9, F8, E-8) +- From that commit every use case that maintains the read model writes the active generation, when there is one, and the one being built through `applyProjection`, in live-created mode for a stream it created and live-updated mode otherwise (D9, E-9) +- Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building`, the phase is not `backfill` or the fence differs, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) +- A batch that throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, and a developer error or a timeout ends the chain; the operator's `interruptGeneration` ends a chain on purpose, and `resumeGeneration` schedules a batch from the saved cursor with a new fence and, when asked, a smaller batch; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) +- Each verify batch enumerates the sources again from the progress row's cursor, which the end of the backfill reset to the first tenant, and calls `applyProjection` in backfill mode for every DTO, so a missing row is inserted, an older one updated, and the row of a DTO that projects to `null` deleted; each such repair is counted as a miss; when the last tenant's last page is done the generation moves to `verified` and no batch is scheduled (D9, E-8, Sc L2-5) +- `switchGeneration` sets the verified generation `active` and, when the read model has an active generation, sets that one `retired` with `retiredAt` and `retireAfter`, in one mutation; every query reads the active generation from the registry, so every subscription on the read model re-runs (D9, F7, F1) +- [extension] During the rollback period the retired generation's rows are kept and not written; `rollbackGeneration` reopens the retired generation it is given as `verifying` with a new fence and a `null` cursor and schedules a verify batch, live commands write it again from that commit, its one verify pass repairs every row that is missing, older or left behind by a deleted subject, and the ordinary `switchGeneration` makes it active and retires the generation that served meanwhile (E-8, D9, Sc L2-5) +- `abortGeneration` marks a building, verifying or verified generation `aborted`, leaves the active generation untouched, and resumes every closed gate entry that names this generation; an aborted rollback leaves the reopened generation `aborted`, never `retired` again (D9, Sc L2-6, E-8) +- `purgeGeneration` starts the purge of an aborted generation, or of a retired one once `retireAfter` has passed; each purge batch deletes at most `limitPurgeBatch` rows of the generation for one tenant by the `by_key` range, stays on that tenant while a full batch came back, moves to the next tenant of the tenant list otherwise, and marks the generation `purged` after the last tenant; nothing starts a purge by itself (D19, E-40, E-49) - For a history-dependent read model the generation row records `pauseRequired`, and `startGeneration` closes the maintenance gate for every source scope of the read model's bindings in the mutation that inserts the row, recording its own generation on each closed entry, so the gate is closed before the first batch is scheduled and this generation's batches are the one background writer allowed through (D9, E-8, E-38) - While the gate is closed for the sources, every use case whose declaration writes one of them and the journal's baseline driver refuse with `writePaused` before any write, whether or not they maintain the view; a batch scheduled before the close reads the closed gate when it starts and refuses, a batch running when the close commits loses the optimistic-concurrency race on the gate document and re-runs into the refusal, and a batch of a superseded chain refuses because its generation fence is stale; nothing needs to drain (D9, D4, E-8, E-38, F1) -- Backfill under the pause enumerates a consistent cut because no source changes: each history batch takes one stream from the enumeration, reads one `history` page of it from the saved event cursor, folds the page into the building generation through `applyHistoryProjection` in backfill mode, and saves the stream and the event cursor on the generation row, moving to the next stream when the page reports `isDone`; an interrupted batch resumes from its cursor with the gate still closed, and the gate never reopens by itself (D9, E-9, Sc L2-6) -- Verification runs under the pause: each verify batch enumerates the sources again and checks, per stream, that the row of its last event's key records the stream's current version in `sourceVersions`, backfilling the stream's history in place when it does not; the operator switches the generation and only then resumes the gate, so writes reopen only after verification (D9, Sc L2-6) -- `abortGeneration` marks a building, verifying or verified generation `aborted`, leaves the active generation untouched, and resumes every closed entry that names this generation; aborted rows are purged in bounded batches (D9, Sc L2-6) +- Backfill under the pause enumerates a consistent cut because no source changes: each history batch takes one stream from the enumeration, reads one `history` page of it from the saved event cursor, folds the page into the building generation through `applyHistoryProjection` in backfill mode, and saves the stream and the event cursor on the progress row, moving to the next stream when the page reports `isDone`; an interrupted batch resumes from its cursor with the gate still closed, and the gate never reopens by itself (D9, E-9, Sc L2-6) +- Verification runs under the pause: each verify batch enumerates the sources again and checks, per stream, that the row of its last event's key records the stream's current version in `sourceVersions`, backfilling the stream's history in place when it does not; `switchGeneration` resumes every closed entry that names the generation in the mutation that makes it active, so writes reopen only after verification (D9, Sc L2-6) ## Design -Each batch is one scheduled internal mutation in the parent. The rebuild schedules from a mutation, so a batch is scheduled only if its predecessor committed, and a scheduled mutation retries internal errors and runs once. The rebuild is an operator operation on parent tables; it is not deferred business work and creates no obligation. All rebuild functions are `internalMutation` or `internalQuery` and reach the dashboard through operator functions that authorize an operator actor; no public function starts, switches or aborts a generation. `Actor` is the pipeline's actor type and `actorValidator` its validator; `rowBudgetBytes` is the projection contract's declared row budget. +Each batch is one scheduled internal mutation in the parent. The rebuild schedules from a mutation, so a batch is scheduled only if its predecessor committed. The rebuild is an operator's work on parent tables; it is not deferred business work and creates no obligation. The library exports one plain helper per entry below, of the same name, taking `(ctx, config: RebuildConfig, args)` and returning the entry's value; a composition's `rebuild` module registers each entry over its helper with its own `internalMutation` and `internalQuery`, so the entries are reached as `internal.rebuild.startGeneration` and so on, and no public function starts or changes a generation. Every refusal is a plain `Error` with the message given; in a message, `name` is the read model's name, `n` the generation number and `state` the generation's state. -The enumeration is the context's `list` query with `includeDeleted`, the one read of context data per batch: one call returns a page of up to `batchSize` DTOs and a continuation cursor, built inside the component by `paginator` from `convex-helpers` over the `by_identity` index that includes deleted subjects, and capped by the query at its byte-derived page bound. A DTO carries the stream version the projection compares, as the queries contract's `toDto(state, meta)` maps it onto `version`. Deleted subjects reach the projection as DTOs the projection maps to `null`, so their rows are deleted rather than left behind. +The enumeration is the context's `list` query with `includeDeleted`, the one read of context data per batch: one call returns a page of up to `batchSize` DTOs and a continuation cursor, built inside the component by `paginator` from `convex-helpers` over the `by_identity` index that includes deleted subjects, and capped by the query at its byte-derived page bound. A batch reads `page`, `isDone` and `continueCursor` and ignores any split status, because the rows up to `continueCursor` are complete. A DTO carries the stream version the projection compares, as the queries contract's `toDto(state, meta)` maps it onto `version`. Deleted subjects reach the projection as DTOs the projection maps to `null`, so their rows are deleted rather than left behind. -- transactionBoundary: each batch is one scheduled internal mutation that commits its rows, its cursor and the scheduling of the next batch together; the switch, rollback and abort are one mutation each (D9, F8, F9) -- convexSurface: `startGeneration`, `backfillBatch`, `resumeGeneration`, `verifyBatch`, `switchGeneration`, `rollbackGeneration`, `abortGeneration`, `purgeGenerationBatch` as internal mutations, `getGenerations` as an internal query, all in the parent; the gate functions are in the write-pause contract (D9, E-8) -- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.number(), batchSize: v.optional(v.number()), pauseRequired: v.boolean(), actor: actorValidator }, returns: v.id("generations"), handler })` computing the row's `batchSize` as `min(args.batchSize ?? 100, floor(4 MiB / (2 * rowBudgetBytes)))` from the projection's declared row budget, inserting the row with the cursor at the first tenant of the application's tenant list and, when `pauseRequired`, calling the write pause's `closeScope` once per distinct `source:` scope of the read model's bindings with the new row's ID before scheduling the first batch; `pauseRequired` is refused with a plain error for a read model whose projection is not a `HistoryProjection`, and required for one that is (D9, E-8, E-9, E-38, E-40) -- fnHistoryPageCall: `ctx.runQuery(components.orders.queries.order.history, { tenantId, streamId, paginationOpts: { numItems: limitHistoryBatch, cursor: eventCursor } })` the one read of context data a history batch makes beside the `list` call of `numItems` 1 that gives it its next stream, returning a `paginator` page of envelopes in version order that the batch folds through `applyHistoryProjection` (D2, D10, E-9, E-24) +- transactionBoundary: each batch is one scheduled internal mutation that commits its rows, its checkpoint and the scheduling of the next batch together; every operator entry is one mutation (D9, F8, F9) +- convexSurface: `startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` as the operator entries, `backfillBatch`, `verifyBatch` and `purgeBatch` as the scheduled batches, all internal mutations, and `getGenerations` as an internal query, in the composition's `rebuild` module; the gate entries are in the write-pause contract (D9, E-8, E-48) +- typeRebuildTarget: `type RebuildTarget = { readModel: ReadModel; source: SourceRef; list: FunctionReference<"query", "internal", ListArgs, PaginationResult> }` one per-entity read model the deployment can rebuild, its one source stream type, and the reference to that stream type's `list` query in its context component, such as `components.orders.queries.order.list` (E-8, E-9, D10) +- typeRebuildConfig: `type RebuildConfig = { targets: readonly RebuildTarget[]; refs: { backfillBatch: BatchRef; verifyBatch: BatchRef; purgeBatch: BatchRef } }` where `type BatchRef = FunctionReference<"mutation", "internal", { generationId: Id<"generations">; fence: number }, null>` and `refs` are the composition's own registered batches, which a batch schedules by reference (E-40, F8) +- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; then it inserts the generation row with the next number, `state: "building"`, `pauseRequired: false`, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `phase: "backfill"`, the batch size, `cursor: null` and every count 0, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48) +- fnBatchSizeFor: `batchSizeFor(readModel: ReadModel, requested?: number): number` throwing `"The batch size must be a whole number of at least 1, and " + requested + " is not"` unless `requested` is absent or an integer of at least 1, and returning `min(requested ?? rebuildBatchCeiling(readModel), rebuildBatchCeiling(readModel))`, so a batch size of 0, a negative one and a fraction are refused and a larger one is lowered to the ceiling (E-40, F13) - fnBackfillBatch: `export const backfillBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9, E-40) -- fnEnumerateCall: `ctx.runQuery(components.orders.queries.order.list, { tenantId, includeDeleted: true, paginationOpts: { numItems: batchSize, cursor } })` returning the `paginator` page `{ page, isDone, continueCursor }` of DTOs with their stream versions, one call per batch; the query caps `numItems` at its byte-derived bound, so a batch never reads more than 8 MiB of stream rows however `batchSize` was set (D2, D10, F15, S4, E-24) -- tenantIteration: `[extension]` every generation spans all tenants, so the generation row's `cursor` and `verifyCursor` are the registry's `BatchCursor`, the current tenant and the page cursor within it; a batch moves to the next tenant of the application's tenant list, read in tenant-ID order, when `isDone`, and the enumeration is done when the last tenant's page is done; a batch never calls `list` without a tenant (E-40, E-8, D11, Law 11) -- fnResumeChain: `resumeChain(ctx: MutationCtx, generationId: Id<"generations">, actor: Actor, batchSize?: number): Promise` bumping the fence, setting the row's `batchSize` when given, at least 1 and at most the bound `startGeneration` computed, clearing `lastError`, and scheduling `backfillBatch` from the saved cursor when the state is `building` or `verifyBatch` from the verify cursor when it is `verifying`, all in the caller's mutation; refused with a plain error in any other state; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) -- fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), batchSize: v.optional(v.number()), actor: actorValidator }, returns: v.null(), handler })` the operator's entry to `resumeChain` (D9, Sc L2-5, E-40) - fnVerifyBatch: `export const verifyBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9) -- fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), actor: actorValidator }, returns: v.null(), handler })` (D9) -- fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { readModel: v.string(), actor: actorValidator }, returns: v.null(), handler })` setting the read model's retired generation to `verifying` with a bumped fence, a cleared verify cursor and zero misses, and scheduling `verifyBatch`; refused when `retireAfter` has passed or when another generation is building, verifying or verified; the switch that completes the rollback is `switchGeneration` (D9, E-8) -- fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), actor: actorValidator }, returns: v.null(), handler })` marking the generation `aborted`, bumping its fence, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8) -- fnPurgeGenerationBatch: `export const purgeGenerationBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` deleting at most `limitPurgeBatch` rows by the `by_key` index range for that generation (D19, E-40) -- step1: `backfillBatch` loads the generation row by ID and returns without writing when `state` is not `building` or `fence` differs from its argument (E-40, D9) -- step2: it calls the write pause's `gateAllows(ctx, scopes, generationId)` with the `all` scope, the scope of the tenant it iterates and the read-model scope, and returns without writing, apart from `writePaused` and the answer's scope in `lastError`, when the answer is not allowed, which it is while the restore door is set and while a closed entry covers one of those scopes and names a generation other than this one; the generation then waits for `resumeGeneration` after the gate reopens; `verifyBatch` and `purgeGenerationBatch` make the same read at the same point, and no batch calls `assertWritable`, because a batch parks its generation rather than failing its scheduled mutation with a thrown refusal (E-8, E-42, D9) -- step3: it calls `list` once with `includeDeleted` and `paginationOpts` of `batchSize` items from the saved cursor (D10, E-40) -- step4: for each item it calls `applyProjection` or `applyAggregate` in backfill mode for the building generation only, counting inserted, updated and skipped (D9, E-9) -- step5: it patches the generation row with the new cursor, the counts and `updatedAt`, and schedules `backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`, or sets `state` to `verifying` and schedules `verifyBatch` when `isDone` (D9, F8) +- fnPurgeBatch: `export const purgeBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D19, E-40) +- fnEnumerateCall: `ctx.runQuery(target.list, { tenantId, includeDeleted: true, paginationOpts: { numItems: batchSize, cursor: pageCursor } })` returning the `paginator` page `{ page, isDone, continueCursor }` of DTOs with their stream versions, one call per batch; the query caps `numItems` at its byte-derived bound, so a batch never reads more than 8 MiB of stream rows however `batchSize` was set (D2, D10, F15, S4, E-24) +- step1: `backfillBatch` loads the generation row by ID and its progress row by `by_generation`, and returns without writing when the row is absent, `state` is not `building`, `phase` is not `backfill` or `fence` differs from its argument (E-40, D9) +- step2: it takes its tenant: the cursor's `tenantId`, or `nextTenant(ctx, null)` when the cursor is `null`; when the tenant list is empty the pass is done and the batch goes to step 6 (E-49, Law 11) +- step3: it calls the write pause's `gateAllows(ctx, ["all", "tenant:" + tenantId], generationId)`, with `["all"]` alone when it has no tenant, and when the answer is not allowed it parks: it patches the progress row's `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt`, schedules nothing and returns, so the generation waits for `resumeGeneration` after the gate reopens; no batch calls `assertWritable`, because a batch parks its generation rather than failing its scheduled mutation with a thrown refusal (E-8, E-42, D9) +- step4: it calls `list` once as `fnEnumerateCall` states, with the cursor's `pageCursor` (D10, E-40) +- step5: for each DTO of the page it calls `applyProjection(ctx, target.readModel, { tenantId, dto, versions: [dto.version], mode: "backfill", generations: [thisGeneration] })` for this generation alone, adding one to `rowsWritten` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` for `skipped-newer` or `unchanged` (D9, E-9) +- step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"`, patches the progress row to `phase: "verify"`, `cursor: null`, `misses: 0` and the counts, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) +- verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `phase` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and the progress row to `phase: "idle"` and `cursor: null`, and schedules nothing (D9, E-8, Sc L2-5) +- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `phase` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(limitPurgeBatch)`, deletes each row and adds their number to `rowsPurged`; when it deleted `limitPurgeBatch` rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`; a pass that is not done checkpoints and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"` and the progress row to `phase: "idle"` and `cursor: null` (D19, E-40, E-49, Law 11) +- purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) +- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `phase` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) +- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string, batchSize?: number): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `phase` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, setting the progress row's `batchSize` to `batchSizeFor(readModel, batchSize)` when one is given, clearing `lastError`, and scheduling the batch of the phase, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) +- fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain` (D9, Sc L2-5, E-40, E-48) +- fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) +- fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `phase` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, the in-flight error of `startGeneration` when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `phase: "verify"`, `cursor: null`, `misses: 0` and no `lastError`, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) +- fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and `"Generation " + n + " of " + name + " is " + state + "; abortGeneration needs building, verifying or verified"` unless the state is one of the three; otherwise patching the row to `state: "aborted"`, fence plus 1, `changedAt` and `changedBy`, patching the progress row to `phase: "idle"` and `lastError: "aborted: " + reason`, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8, E-48) +- fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged"` when `phase` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `phase: "purge"`, `cursor: null` and no `lastError`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) +- fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(v.object({ generation: generationDoc, progress: progressDoc })), handler })` the newest `limitGenerationsListed` generations of the read model by `by_read_model` in descending order, each with its progress row, where `generationDoc` and `progressDoc` are the two tables' document validators with `_id` and `_creationTime`; an operator entry that only reads, so it takes no operator (E-8, E-40, E-48, D19) +- errorUnknownGeneration: every entry and every helper that takes a `generationId` throws `"No generation " + generationId` when the row is absent, and a batch returns without writing instead (E-8) +- stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its phase is not `idle`, and the operator answers with `resumeGeneration` and a smaller batch, down to 1 (E-40, F1, F9, Probe 6) +- occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch; a batch reads the generation row and never writes it outside the end of a pass, and it writes the progress row, which no command reads, so a batch's checkpoint re-runs no command (F1, D9, Probe 6, E-40) +- limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) +- limitPurgeBatch: `min(500, floor(4194304 / rowBudgetBytes))` rows deleted per purge batch, 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most 4 MiB (E-40, F13, D19) +- limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch returns without writing (E-40, F13) +- limitBuildingGenerations: at most one generation in `building`, `verifying` or `verified` per read model at a time, so a command writes at most two generations, the active one and that one (E-8, E-9) +- limitRollbackPeriod: `[extension]` 7 days by default, stored as `retireAfter` on the retired row; the retired generation is kept and not written, and a rollback at or after `retireAfter` is refused (E-8, D9) +- limitGenerationsListed: 20 generations per `getGenerations` answer, the newest first (E-8, F13) +- fnHistoryPageCall: `ctx.runQuery(components.orders.queries.order.history, { tenantId, streamId, paginationOpts: { numItems: limitHistoryBatch, cursor: eventCursor } })` the one read of context data a history batch makes beside the `list` call of `numItems` 1 that gives it its next stream, returning a `paginator` page of envelopes in version order that the batch folds through `applyHistoryProjection` (D2, D10, E-9, E-24) - historyStep3: for a history view a batch replaces steps 3 and 4: when the cursor names no stream it calls `list` with `numItems` 1 from the page cursor to take the next stream and sets `eventCursor` to `null`; it then calls `history` for that stream with `limitHistoryBatch` events from `eventCursor`, calls `applyHistoryProjection` in backfill mode for the building generation with the page, and saves `eventCursor`, or clears `streamId` and advances the page cursor when the page reports `isDone` (D9, E-9, E-24, E-40) - historyVerify: a verify batch of a history view enumerates the sources with `list` and, per stream, reads its last event through `history` with `fromVersion` at the stream's version and `numItems` 1, finds that event's row by `keyOf`, and counts a miss when the row is absent or its `sourceVersions` record a lower version for the stream; a miss is repaired by folding the stream's history in place through the same history batches from `eventCursor` null (D9, E-9, Sc L2-6) -- migrationsBinding: `[extension] const migrations = new Migrations(components.migrations, { internalMutation })` with `migrations.define({ table: "orderSummaries", batchSize: 100, customRange: (q) => q.withIndex("by_key", (i) => i.eq("tenantId", tenantId).eq("generation", generation)), migrateOne })` for batches whose source is a parent table, such as purging a generation or re-projecting an existing generation's rows; the component resumes from the failed batch (E-40, F17) -- rebuildClass: a read model declares its class at declaration time, per entity or bounded current state for the online path, as a `Projection` or `AggregateProjection`, and history across streams for the write pause, as a `HistoryProjection`; `pauseRequired` on the generation row records it, and a history view never rebuilds online because a live fold onto a row backfill has not reached cannot be told apart from a complete one (D9, D18, E-9) -- limitHistoryBatch: [extension] `min(500, floor(4 MiB / (2 * rowBudgetBytes)))` events per history batch, 128 at the default 16 KiB row budget and 32 at the 64 KiB ceiling, so one batch reads at most that many events of at most `limitEventBytes` of `spec:context.event-envelope`, 20,480 bytes and the 61 bytes of a stored document's system fields, about 2.5 MiB, one stream row through `list`, and at most one history row per event, at most 2 MiB, and writes at most 2 MiB, under the ceilings with the 1 second timeout in mind; the count is stored on the generation row as its `batchSize` and lowered by `resumeGeneration` like the online bound (E-40, E-9, E-24, F13) -- occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch, and a batch that keeps losing fails after Convex's internal retries with its documents-changed error, which is the stall that `resumeGeneration` answers with a smaller bound (F1, D9, Probe 6, E-40) -- markerRule: an aggregate write reads the entity's marker for the generation in the same mutation and adds only the difference between the new and the recorded contribution, so a live command and a backfill batch that both see the entity never count it twice (D9, E-8) -- limitBatchSize: at most 100 entities per backfill or verify batch, this design's own bound under E-40 and F13 until Probe 6, and at most `floor(4 MiB / (2 * rowBudgetBytes))` for the read model's declared row budget, which is 128 and so 100 at the default 16 KiB and 32 at the 64 KiB ceiling of `limitRowSize`, computed by `startGeneration`, stored as `batchSize` and passed to a migrations-driven batch so that both runners carry the same number; the context's `list` caps the page further at `limitListPage(budgetBytes)`, `max(1, min(100, floor(4 MiB / budgetBytes)))` items, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; lowered per generation down to 1 by `resumeGeneration`; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, the generation row and at most two point reads per entity, the building generation's row and, for an aggregate, its marker, each at most `rowBudgetBytes`, at most 4 MiB, and at most two writes per entity plus the generation row, at most 4 MiB written, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, F17, Probe 4) -- limitPurgeBatch: 500 rows deleted per purge batch, one index range, under 1 second (E-40, F13, D19) -- limitOutstandingBatches: one scheduled batch per generation at a time, enforced by the fence; a stale batch returns without writing (E-40, F13) -- limitBuildingGenerations: at most one generation in `building`, `verifying` or `verified` per read model at a time, so a command writes at most two generations, the active one and that one (E-8, E-9) -- limitRollbackPeriod: `[extension]` 7 days by default, stored as `retireAfter` on the retired row; the retired generation is kept but not written, and a rollback after `retireAfter` is refused (E-8, D9) +- rebuildClass: a read model declares its class at declaration time, per entity for the online path, as a `ReadModel` with its projections, and history across streams for the write pause, as a `HistoryProjection`; `pauseRequired` on the generation row records it, and a history view never rebuilds online because a live fold onto a row backfill has not reached cannot be told apart from a complete one (D9, D18, E-9) +- limitHistoryBatch: [extension] `min(500, floor(4 MiB / (2 * rowBudgetBytes)))` events per history batch, 128 at the default 16 KiB row budget and 32 at the 64 KiB ceiling, so one batch reads at most that many events of at most `limitEventBytes` of `spec:context.event-envelope`, 20,480 bytes and the 61 bytes of a stored document's system fields, about 2.5 MiB, one stream row through `list`, and at most one history row per event, at most 2 MiB, and writes at most 2 MiB, under the ceilings with the 1 second timeout in mind; the count is stored on the progress row as its `batchSize` and lowered by `resumeGeneration` like the online bound (E-40, E-9, E-24, F13) ## Example space @@ -144,9 +162,11 @@ And writes {writes:"never stopped"|"reopen only after verification"|"reopen afte ## Verification — reviewed - A reviewer confirms that every batch function states its bound and that the bounds sit below the F13 ceilings with the 1 second timeout in mind. -- A reviewer confirms that no rebuild function calls a context operation, schedules an action or writes anything but read-model rows, markers, the generation row and the gate. +- A reviewer confirms that no rebuild function calls a context operation, schedules an action or writes anything but read-model rows, the generation row, the progress row and the gate. - A reviewer confirms that the version compare in `applyProjection` is the only thing that prevents a stale overwrite and that L2-5's example asserts it against a concurrent live command. -- A reviewer confirms that the gate is read inside every writer's mutation, not cached, that every batch reads it through `gateAllows` and parks its generation instead of throwing, that `startGeneration` with `pauseRequired` closes one entry per source of the view and nothing else, and that the write-pause examples assert a refusal of a source writer while closed, the fold of the history view against the journal, and a reopen only after verification. -- A reviewer confirms that no registry function, batch or query takes a tenant scope for a generation, that every batch names the tenant it iterates on its context call, and that L2-5's example seeds more than one tenant and asserts both are covered at cutover. +- A reviewer confirms that a batch patches the generation row only in the mutation that ends its pass, and that no command or query over a read model reads the progress row. +- A reviewer confirms that every entry that changes a generation calls `assertOperator` first and writes the operator on the row, and that no rebuild function reads a grant or `ctx.auth`. +- A reviewer confirms that the gate is read inside every writer's mutation, not cached, that every batch reads it through `gateAllows` and parks its generation instead of throwing, that `startGeneration` closes one entry per source of a paused view and nothing else, and that the write-pause examples assert a refusal of a source writer while closed, the fold of the history view against the journal, and a reopen only after verification. +- A reviewer confirms that no registry function, batch or query takes a tenant scope for a generation, that every batch takes its tenant from `nextTenant` and names it on its context call, and that L2-5's example seeds more than one tenant and asserts both are covered at cutover. - A reviewer confirms that `resumeChain` is the only code path that reschedules a stranded generation and that the restore's `finishRestoreRun` calls it for every generation its inventory lists in `building` or `verifying`. -- A reviewer confirms that no function writes a retired generation, that `rollbackGeneration` is the only transition out of `retired` besides purge, and that L2-5's example asserts the reopened generation is `verified` before it is active. +- A reviewer confirms that no function writes a retired generation, that `rollbackGeneration` is the only transition out of `retired` besides purge, and that L2-5's example asserts that the reopened generation is `verified`, holds a row for a subject created since its retirement and none for a subject deleted since, before it is active. diff --git a/design/specs/application/rebuild.write-pause-rebuild-abort.sdp.md b/design/specs/application/rebuild.write-pause-rebuild-abort.sdp.md index 50d6202..c4cb1e7 100644 --- a/design/specs/application/rebuild.write-pause-rebuild-abort.sdp.md +++ b/design/specs/application/rebuild.write-pause-rebuild-abort.sdp.md @@ -33,4 +33,4 @@ And writes {writes: "reopen after the abort"} - The view is the order allocation history of `spec:application.orders-inventory-example`, a `HistoryProjection` bound to the order stream type and the stock item stream type. - The test starts a generation with `pauseRequired`, asserts that the gate holds one closed `source:` entry for each of the two stream types, each naming this generation, interrupts after the second history batch and calls `abortGeneration`. - The test asserts that the generation is `aborted`, that the active generation is unchanged, that both source entries are gone from the gate, that `PlaceOrder` succeeds again, and that no query returns a row of the aborted generation. -- The test runs `purgeGenerationBatch` until the generation is `purged` and asserts every batch deleted at most 500 rows. +- The test runs `purgeGeneration` and asserts that its batches run until the generation is `purged` and that every batch deleted at most `limitPurgeBatch` rows. diff --git a/design/specs/application/restore.sdp.md b/design/specs/application/restore.sdp.md index 70eb1dd..43ea176 100644 --- a/design/specs/application/restore.sdp.md +++ b/design/specs/application/restore.sdp.md @@ -32,7 +32,7 @@ relations: --- # Restore -Layer 2 · Detail: full · Traces: D3, D5, D6, D8, D19, Law 3, Law 8, Law 10, F8, F12, F13, S8, Probe 7, Sc L2-8, E-42. +Layer 2 · Detail: full · Traces: D3, D5, D6, D8, D19, Law 3, Law 8, Law 10, F8, F12, F13, S8, Probe 7, Sc L2-8, E-42, E-48. Restore is the procedure that puts a backup's data under matching code and configuration and proves, before any writer runs, that the domain, journal and read-model invariants hold. A backup is a consistent snapshot of table data; it excludes code, configuration, environment variables and pending scheduled functions, and restoring it wipes the deployment's data and replaces it. The procedure therefore starts by closing the door to writers through the environment, deploys the matching commit, restores, runs its checks as bounded batches, takes an inventory of the pending work the snapshot carries but the scheduler no longer drives, reconciles the gap since the backup, and reopens only when every check passes. Because the snapshot holds the maintenance gate and the generation rows as they were at backup time but not the scheduled batch that was driving a rebuild, a generation left in `building` or `verifying` is stranded by the restore, and a closed gate entry that names it would keep its writers out for ever; the inventory lists every such row and the accepted branch resumes each generation the operator did not abort. Restore is tested, not only backup: the drill is the acceptance scenario, run on a representative dataset in a disposable deployment. @@ -91,17 +91,17 @@ The restore itself is a platform operation outside any mutation. Everything the - transactionBoundary: none for the snapshot restore; each check batch is one internal mutation that reads through one component query and writes only the run record; `finishRestoreRun` is one mutation that marks the run and reschedules the stranded generations together, so acceptance and the rescheduling commit or fail as one (D19, F8, F13) - convexSurface: `startRestoreRun`, `streamCheckBatch`, `readModelCheckBatch`, `receiptCheckBatch`, `pendingWorkBatch`, `finishRestoreRun` as internal mutations and `getRestoreRun` as an internal query, all in the parent (E-42, D19) -- tableRestoreRuns: `restoreRuns: defineTable({ backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), state: v.union(v.literal("running"), v.literal("accepted"), v.literal("failed")), checks: v.array(v.object({ name: v.string(), cursor: v.optional(v.string()), examined: v.number(), findings: v.array(v.object({ subject: v.string(), detail: v.string() })), state: v.union(v.literal("pending"), v.literal("running"), v.literal("passed"), v.literal("failed")) })), pendingWork: v.array(pendingWorkItemValidator), operator: actorValidator, startedAt: v.number(), finishedAt: v.optional(v.number()) })` where `checks` holds five entries, the four checks and the inventory under the name `pendingWork`, whose cursor and state are kept like a check's while its items go to `pendingWork` (E-42, D19) +- tableRestoreRuns: `restoreRuns: defineTable({ backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), state: v.union(v.literal("running"), v.literal("accepted"), v.literal("failed")), checks: v.array(v.object({ name: v.string(), cursor: v.optional(v.string()), examined: v.number(), findings: v.array(v.object({ subject: v.string(), detail: v.string() })), state: v.union(v.literal("pending"), v.literal("running"), v.literal("passed"), v.literal("failed")) })), pendingWork: v.array(pendingWorkItemValidator), operator: v.string(), startedAt: v.number(), finishedAt: v.optional(v.number()) })` where `checks` holds five entries, the four checks and the inventory under the name `pendingWork`, whose cursor and state are kept like a check's while its items go to `pendingWork` (E-42, D19) - validatorPendingWorkItem: `const pendingWorkItemValidator = v.object({ kind: v.union(v.literal("generation"), v.literal("closedScope"), v.literal("import")), subject: v.string(), detail: v.string(), generationId: v.optional(v.id("generations")), fence: v.optional(v.number()), resolution: v.optional(v.union(v.literal("resumed"), v.literal("aborted"), v.literal("kept"), v.literal("recorded"))) })` where `subject` is the read model and generation number, the scope key, or the tenant and import ID, `fence` is the generation's fence at inventory time, and `resolution` is written by `finishRestoreRun` (E-42, D19) - validatorCheckScope: `const checkScopeValidator = v.union(v.object({ kind: v.literal("tenant"), tenantId: v.string() }), v.object({ kind: v.literal("all") }))` naming which tenants a check covers; under `all` a check batch iterates the application's tenant list the way the rebuild's `tenantIteration` does, one tenant named on every context call, and never calls a context without a tenant (E-42, D11, Law 11) - indexRestoreRunsByStarted: `.index("by_started", ["startedAt"])` (E-42) - indexRestoreRunsByStartedUse: the operator's listing of runs, newest first (E-42, D19) -- fnStartRestoreRun: `export const startRestoreRun = internalMutation({ args: { backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), operator: actorValidator }, returns: v.id("restoreRuns"), handler })` refusing when `MAINTENANCE_MODE` is not `restore` (E-42, D19) +- fnStartRestoreRun: `export const startRestoreRun = internalMutation({ args: { backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), operator: v.string() }, returns: v.id("restoreRuns"), handler })` an operator entry of `spec:command.actor-and-scope`: it calls `assertOperator` first and records the stated operator on the run record, refusing when `MAINTENANCE_MODE` is not `restore` (E-42, E-48, D19) - fnStreamCheckBatch: `export const streamCheckBatch = internalMutation({ args: { runId: v.id("restoreRuns"), contextId: v.string(), streamType: v.string(), scope: checkScopeValidator }, returns: v.boolean(), handler })` calling `ctx.runQuery(components.orders.queries.order.list, { tenantId, includeDeleted: true, paginationOpts: { numItems: streamsPerBatch, cursor } })` once, where `streamsPerBatch` is the stream bound of `limitCheckBatch`, and, per stream of the page in order, `ctx.runQuery(components.orders.queries.order.rebuild, { tenantId, streamId, maxEvents: maxFoldEvents - sum })` where `sum` is the running total of `eventsApplied` in this batch; the query reads the stream row first and refuses with `budgetExhausted` before reading an event when the tail exceeds `maxEvents`, on which the batch saves the cursor at that stream and returns, so the next batch starts with `sum` at zero and admits the same stream alone; a `tailTooLong` refusal is a journal finding, `equal` false a domain finding, and the batch returns whether the enumeration is done (E-42, E-24, E-25, D2, D3, D5, Sc L0-2) - fnReadModelCheckBatch: `export const readModelCheckBatch = internalMutation({ args: { runId: v.id("restoreRuns"), readModel: v.string(), scope: checkScopeValidator }, returns: v.boolean(), handler })` reusing the rebuild's verify logic in compare mode against the active generation (E-42, D8, D9) - fnReceiptCheckBatch: `export const receiptCheckBatch = internalMutation({ args: { runId: v.id("restoreRuns"), scope: checkScopeValidator }, returns: v.boolean(), handler })` reading 25 receipts by tenant from the receipt cursor and calling `ctx.runQuery(components.orders.queries.operations.byOperation, { tenantId, operationId })` per receipt for each context the receipt's versions name (E-42, D6, D19) - fnPendingWorkBatch: `export const pendingWorkBatch = internalMutation({ args: { runId: v.id("restoreRuns") }, returns: v.boolean(), handler })` which on its first call, when the inventory's cursor is empty, reads the gate document and appends one `closedScope` item per closed entry, then reads `by_read_model_state` for each read model the parent's declarations name and each of `building`, `verifying` and `verified` and appends one `generation` item per row with its state and fence, and on every call pages the `imports` table with `.paginate` in creation order, `limitPendingWorkBatch` rows at a time, appending one `import` item per row in `running`; it saves the page cursor on its `checks` entry and returns whether the paging is done (E-42, F12, D19, E-41) -- fnFinishRestoreRun: `export const finishRestoreRun = internalMutation({ args: { runId: v.id("restoreRuns"), operator: actorValidator }, returns: v.null(), handler })` marking the run failed when any check has a finding; otherwise refusing with a plain error while `MAINTENANCE_MODE` is still `restore` or the inventory has not finished, and then, for every `generation` item whose row is still in `building` or `verifying` at the inventoried fence, calling the rebuild's `resumeChain` in this mutation, which bumps the fence and schedules the next batch from the saved cursor, writing `resumed` on the item, `aborted` on one whose row is `aborted`, `kept` on a `closedScope` item still closed, `recorded` on an `import` item, and marking the run accepted with the finish time (E-42, D19, F8, F12, Sc L2-8) +- fnFinishRestoreRun: `export const finishRestoreRun = internalMutation({ args: { runId: v.id("restoreRuns"), operator: v.string() }, returns: v.null(), handler })` an operator entry of `spec:command.actor-and-scope`, which calls `assertOperator` first and passes the stated operator to `resumeChain`, marking the run failed when any check has a finding; otherwise refusing with a plain error while `MAINTENANCE_MODE` is still `restore` or the inventory has not finished, and then, for every `generation` item whose row is still in `building` or `verifying` at the inventoried fence, calling the rebuild's `resumeChain` in this mutation, which bumps the fence and schedules the next batch from the saved cursor, writing `resumed` on the item, `aborted` on one whose row is `aborted`, `kept` on a `closedScope` item still closed, `recorded` on an `import` item, and marking the run accepted with the finish time (E-42, E-48, D19, F8, F12, Sc L2-8) - doorExemption: `startRestoreRun`, `streamCheckBatch`, `readModelCheckBatch`, `receiptCheckBatch`, `pendingWorkBatch` and `finishRestoreRun` do not read the gate through `assertWritable` or `gateAllows` and write only `restoreRuns`, except that `finishRestoreRun`'s accepted branch also patches the generation rows it resumes; `startRestoreRun` refuses when `MAINTENANCE_MODE` is not `restore`, and each later mutation refuses when its run is not `running`, so no operational row is written outside a restore and no business table is written by them at all (E-42, D19) - checkOrder: journal, domain, read model, receipt, then the inventory; a later check does not start while an earlier one has findings, because a broken journal makes the fold check meaningless, and the inventory runs last because it is not a check and its items never fail the run (E-42, D3) - gapRecord: the run record's `backupTakenAt` is the gap start; the procedure writes no synthetic receipt and invents no event for the gap (D19, Existing systems) diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index 7d0f47f..e37e8d2 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -18,63 +18,72 @@ relations: --- # Write pause -Story · Detail: full · Traces: D4, D9, D19, Law 6, F1, F9, F12, F13, Sc L2-6, Sc L2-8, E-8, E-42. +Story · Detail: full · Traces: D4, D9, D19, Law 6, F1, F9, F12, F13, Sc L2-6, Sc L2-8, E-8, E-42, E-43, E-48. -The write pause is the maintenance gate every writer in a scope reads inside its own mutation. It exists for the rebuild of a read model that depends on history across several streams, where a consistent cut is needed, and the restore procedure holds the same door closed for every scope through the environment, because the snapshot replaces the gate document, until its checks pass. A closed gate turns every write in scope into a transient refusal, retryable and never stored; a background batch scheduled before the close reads the closed gate when it starts, and a batch of a superseded chain is refused by its generation's fence. The rebuild closes the gate for its own scope in the mutation that registers its generation, the operator closes a tenant or the whole deployment, and the operator resumes and aborts. Nothing reopens the gate by itself. Optimistic concurrency makes the close a consistent cut, so the gate has one closed state and no drain. +The write pause is the maintenance gate every writer in a scope reads inside its own mutation. It exists for the rebuild of a read model that depends on history across several streams, where a consistent cut is needed, and the restore procedure holds the same door closed for every scope through the environment, because the snapshot replaces the gate document, until its checks pass. A closed gate turns every write in scope into a transient refusal, retryable and never stored; a background batch scheduled before the close reads the closed gate when it starts, and a batch of a superseded chain is refused by its generation's fence. The rebuild closes the gate for its own scope in the mutation that registers its generation, an operator entry closes a tenant or the whole deployment, and operator entries resume and abort. Nothing reopens the gate by itself. Optimistic concurrency makes the close a consistent cut, so the gate has one closed state and no drain. ## Intent -- outcome: Pin the maintenance gate document, the writer check every mutation in scope makes, the fencing of background writers, and the operator's close, resume and abort (D9, E-8) +- outcome: Pin the maintenance gate document, the writer check every mutation in scope makes, the fencing of background writers, and the operator entries that close, resume and read the gate (D9, E-8) - value: One document read per command gives the rebuild its consistent cut and the restore its closed door, with no lock protocol and no queue (D9, D1) - risk: Every use case reads the gate document, one point read per command, and a gate change makes every concurrent command re-run once under optimistic concurrency (F1, E-8) +- risk: This contract states no acceptable length for a write pause; before a paused rebuild starts, its length is estimated as its backfill batches times the time of one batch plus its verify batches times the time of one batch, with the batch time taken from the first paused rebuild measured on the native backend, and whether that length is the trigger for an online history rebuild is ruled with that number and not before it (D9, D18) - assumption: Mutations are serializable under optimistic concurrency, so a writer that read the gate as open cannot commit after a close that committed first (F1) +- assumption: A mutation reads a deployment's environment variable through `process.env`, and a change of the variable is seen by the next function run without a deploy; no probe of the corpus has shown the second half on a native backend (E-42, F12) ### Open questions -- [non-blocking] Extension E-8: the doc names a maintenance gate every writer reads, fencing or draining of background writers, and operator resume and abort; the option taken here is one gate document per deployment holding the closed scopes, each in one `closed` state, with no fence of its own and no drain interval, because optimistic concurrency serializes every writer's gate read against the close and the generation row's fence already fences a rebuild's batches, and a refusal answered with `writePaused`, the transient code the outcome boundary reserves for this contract under its E-35, which this contract confirms; the scope a rebuild closes is the paused view's sources, one `source:` entry per context and stream type its bindings name, rather than the read model itself, because a consistent cut must hold every writer of those streams and a use case that writes a source stream without maintaining the view would otherwise pass the gate, which is why the command declaration names what it writes under E-38 and the pipeline proves the list at its step 9; the alternative, closing the tenant or `all` for every paused rebuild, is safe and coarser and stays the operator's manual option through `closeGate` (E-8, E-38, D9, D4, F1) +- [non-blocking] Extension E-8: the doc names a maintenance gate every writer reads, fencing or draining of background writers, and operator resume and abort; the option taken here is one gate document per deployment holding the closed scopes, each in one `closed` state, with no fence of its own and no drain interval, because optimistic concurrency serializes every writer's gate read against the close and the generation row's fence already fences a rebuild's batches, and a refusal answered with `writePaused`, the transient code the outcome boundary reserves for this contract under its E-35, which this contract confirms; the scope a paused rebuild closes is the paused view's sources, one `source:` entry per context and stream type its bindings name, because a consistent cut must hold every writer of those streams and a use case that writes a source stream without maintaining the view would otherwise pass the gate, which is why the command declaration names what it writes under E-38 and the pipeline proves the list at its step 9; an operator entry closes only a tenant or `all`, the coarser and always safe scopes, and no scope names a read model, because nothing would close it (E-8, E-38, D9, D4, F1) ## Contract -- There is one gate document per deployment, read by the command pipeline at its step 7 for every command through `assertWritable`, which throws, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes while the restore door is closed, as `spec:application.restore` states (D9, E-8, E-25, E-42) -- A scope is `all`, one tenant, one read model, or one source stream type of one context; the gate holds the set of closed scopes, each with a reason, the generation it serves if any, and who changed it (E-8, D9) -- A writer in a closed scope refuses before any write with the transient refusal `writePaused`, marked retryable, carrying the scope and the reason; nothing is stored (D9, D4, Law 6) -- A use case is in scope when `all` is closed, when its tenant is closed, when a read model it maintains is closed, or when a stream type its declaration's `writes` names is closed (E-8, E-38, D9) +- There is one gate document per deployment, read by the command pipeline at its step 7 for every command through `assertWritable`, which throws, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes while the restore door is closed, as `spec:application.restore` states (D9, E-8, E-25, E-42) +- A scope is `all`, one tenant, or one source stream type of one context; the gate holds the set of closed scopes, each with a reason, the generation it serves if any, and the operator who closed it (E-8, E-48, D9) +- A writer in a closed scope refuses before any write with the transient refusal `writePaused`, whose message carries the scope and the reason; nothing is stored, no operation ID is minted, and the same request key succeeds once the scope is open (D9, D4, Law 6) +- A use case is in scope when `all` is closed, when its tenant is closed, or when a stream type its declaration's `writes` names is closed (E-8, E-38, D9) +- The restore door is read before the gate document: while the deployment's `MAINTENANCE_MODE` environment variable is exactly `restore`, every gate read answers closed with the scope `all` and the reason `restore` and reads no document; any other value, and an unset variable, leave the answer to the gate document (D19, F12, Sc L2-8, E-42) +- A deployment with no gate document is open: the gate read never writes, and the first close inserts the document (E-8, D9) - [extension] A rebuild that needs a consistent cut closes the sources of the paused view, one `source:` scope per context and stream type its bindings name, so every writer of those streams is held whether or not it maintains the view; the closed entries name the generation, and `abortGeneration` and the switch's resume reopen every entry that names it (E-8, E-38, D9, Sc L2-6) -- Background writers are fenced, not drained: a batch scheduled before the close reads the closed gate when it starts and refuses, a batch running when the close commits loses the optimistic-concurrency race on the gate document and re-runs into the refusal, a batch that committed first wrote before the cut, and a batch of a superseded chain returns without writing because the generation row's fence, bumped by every resume and abort, differs from its argument (D9, E-8, F1) +- [extension] The rebuild a scope was closed for is the one background writer allowed through it: a gate read that states a generation passes a closed entry only when the entry names a generation and names that one, so an entry with no generation admits no one, and a gate read that states no generation passes no closed entry (E-8, D9) +- Background writers are fenced, not drained: a batch scheduled before the close reads the closed gate when it starts and refuses, a batch running when the close commits loses the optimistic-concurrency race on the gate document and re-runs into the refusal, a batch that committed first wrote before the cut, and a batch of a superseded chain returns without writing because the generation row's fence differs from its argument (D9, E-8, F1) - The gate has one `closed` state per scope and no draining state, drain interval or confirmation step, because the serialization above already gives the consistent cut and a second state would only add an operator duty (E-8, F1) -- The rebuild the gate was closed for is the one background writer allowed through, identified by its generation on the closed scopes; `startGeneration` closes one entry per source of the view and records its own generation on each in the mutation that inserts the generation row, so every entry names a row that exists (D9, E-8) -- The operator resumes a scope only after the rebuild it served is verified and switched, or aborted; a resume during a build is refused by the operator function unless the operator states abort (D9, Sc L2-6) -- Abort marks the generation aborted and resumes the scope in one mutation (D9, Sc L2-6) -- The restore procedure cannot use the gate document, because the snapshot replaces it; the same writer check also refuses every write while the deployment's `MAINTENANCE_MODE` environment variable is `restore`, which survives a restore because backups exclude environment variables; the restore procedure's own mutations, which write only `restoreRuns` and the generation rows the accepted branch resumes, are the one exemption, and that workflow states it (D19, F12, Sc L2-8, E-42) -- [extension] A closed entry the snapshot brings back is as closed as one written live: an entry that names a generation stays closed until the restore's pending-work step resumes that generation or the operator aborts it, which resumes the scope, and an operator-closed tenant or `all` entry stays closed until the operator resumes it, so the gate never reopens by itself through a restore either, and the restore's accepted branch is what keeps a restored entry from outliving the rebuild it served (E-8, E-42, F12, D19) -- [extension] Gate changes are treated as mandatory audit: every close, resume and abort writes an audit record inside the same mutation and fails closed, because a gate change stops or reopens every writer in a scope and an operator must be able to find who did it; the doc fails closed only for mandatory business or security audit and does not name gate changes among them (E-8, D19) +- `startGeneration` closes one entry per source of a paused view and records its own generation on each in the mutation that inserts the generation row, so every entry that names a generation names a row that exists (D9, E-8) +- [extension] The operator entries are `closeGate` and `resumeGate` for the scope `all` and for a tenant scope, and `getGate` and `getGateAudit` to read; each follows the operator entry contract of `spec:command.actor-and-scope`, and a source scope is closed and reopened only by the rebuild (E-8, E-48, D19) +- Abort marks the generation aborted and resumes its scopes in one mutation (D9, Sc L2-6) +- The restore procedure cannot use the gate document, because the snapshot replaces it; the restore door keeps every writer out through the environment, which survives a restore because backups exclude environment variables; the restore procedure's own mutations, which write only `restoreRuns` and the generation rows the accepted branch resumes, are the one exemption, and that workflow states it (D19, F12, Sc L2-8, E-42) +- [extension] A closed entry the snapshot brings back is as closed as one written live: an entry that names a generation stays closed until the restore's pending-work step resumes that generation or an operator aborts it, which resumes the scope, and a tenant or `all` entry stays closed until an operator resumes it, so the gate never reopens by itself through a restore either (E-8, E-42, F12, D19) +- [extension] Gate changes are treated as mandatory audit: every close and every resume inserts one record through `writeOperatorAudit` of `spec:operations.baseline-operations` inside the same mutation and fails closed, so a gate change whose audit insert fails changes nothing: the gate document is as it was and no record exists; the doc fails closed only for mandatory business or security audit and does not name gate changes among them (E-8, E-43, D19) - A local reaction refused by the gate is delayed, not counted: the obligation wrapper reschedules the same attempt without incrementing its attempt count, as `spec:obligations.local-reaction-wrapper` rules, so a pause never turns pending work into needs attention (D13, D9, E-51) ## Design -The gate is one document so that a command pays one point read. Scopes are strings so that the same document serves tenants, read models and the whole deployment without a table per kind. The refusal is a `ConvexError` whose data is the outcome boundary's `TransientData`, `kind` equal to `"transient"` and `code` equal to `"writePaused"`, the shape the boundary gives every transient refusal. `Actor` is the pipeline's actor type and `actorValidator` its validator. +The gate is one document so that a command pays one point read. Scopes are strings so that the same document serves tenants, source stream types and the whole deployment without a table per kind. The refusal is a `ConvexError` whose data is the outcome boundary's `TransientData`. The table is `gateTables`, a schema fragment the library exports, and `GateDataModel` is its data model; every composition that registers a command spreads it into its schema, because step 7 reads it. The library exports the four operator entries as registered internal functions, built with `internalMutationGeneric` and `internalQueryGeneric` of `convex/server`, and a composition's `gate` module re-exports them, so they are reached as `internal.gate.closeGate`, `internal.gate.resumeGate`, `internal.gate.getGate` and `internal.gate.getGateAudit`. Every refusal of an operator entry is a plain `Error` with the message given. -- tableMaintenanceGates: `maintenanceGates: defineTable({ key: v.literal("gates"), closed: v.array(v.object({ scopeKey: v.string(), reason: v.string(), generationId: v.optional(v.id("generations")), changedAt: v.number(), changedBy: actorValidator })), updatedAt: v.number() })` (E-8, D9) -- indexGatesByKey: `.index("by_key", ["key"])` (E-8) -- indexGatesByKeyUse: the one point read every writer makes; the document is created open on first use (E-8, D9) -- typeScopeKey: `type ScopeKey = "all" | string` where a tenant scope is written `tenant:` followed by the tenant ID, a read-model scope `read-model:` followed by the read model name, and a source scope `source:` followed by the context ID, a colon and the stream type, built only by `scopesOfUseCase`, `startGeneration`, the journal's baseline driver and the operator functions (E-8, E-38, D11) -- fnGateAllows: `gateAllows(ctx: MutationCtx, scopes: ScopeKey[], allowGeneration?: Id<"generations">): Promise<{ allowed: true } | { allowed: false; scopeKey: ScopeKey; reason: string }>` the one gate read: it reads `process.env.MAINTENANCE_MODE` and then the gate document once, and answers not allowed with the scope `all` and the reason `restore` while the variable is `restore`, and with the first closed entry that covers one of `scopes` unless that entry's `generationId` equals `allowGeneration`; a background batch calls it directly and parks when the answer is not allowed, as the rebuild's `step2` states, because a scheduled mutation that threw the refusal would end its chain instead of waiting (E-8, E-42, D9, F9) -- fnAssertWritable: `assertWritable(ctx: MutationCtx, scopes: ScopeKey[]): Promise` calling `gateAllows` with no generation and throwing `writePaused` with the answer's scope and reason when it is not allowed; the command pipeline's step 7 calls it, and nothing else does (E-8, D9) -- restoreDoor: `gateAllows` runs in the parent and reads `process.env.MAINTENANCE_MODE` before the gate document; the value `restore` refuses every write with `writePaused` and the scope `all`, so the restore procedure keeps writers out even though the snapshot replaced the gate, and parks every rebuild batch until the accepted branch resumes it; the procedure's own six mutations do not call `gateAllows` or `assertWritable` and write only `restoreRuns` and the generation rows they resume, as `spec:application.restore` states; no component reads the variable (E-42, D19, F11) -- fnScopesOfUseCase: `scopesOfUseCase(tenantId: string, readModels: string[], writes: SourceRef[]): ScopeKey[]` returning `all`, the tenant scope, one read-model scope per maintained read model and one source scope per `SourceRef` of the declaration's `writes`, so a command is held by a paused rebuild of any view whose sources it writes (E-8, E-38, D9) -- errorCodeWritePaused: `writePaused`, the transient code the outcome boundary reserves, thrown through the boundary's `refuseTransient({ code: "writePaused", message: "write paused for " + scopeKey + ": " + reason })`, so its data is `TransientData` with `kind` equal to `"transient"`; a transient refusal for capacity, never a stored rejection (D4, Law 6, E-35) -- fnCloseScope: `closeScope(ctx: MutationCtx, scopeKey: ScopeKey, reason: string, actor: Actor, generationId?: Id<"generations">): Promise` adding the scope to `closed` and writing the audit record in the caller's mutation; called by `closeGate` and by the rebuild's `startGeneration`, which passes the ID of the generation row it has just inserted (E-8, D9, D19) -- fnCloseGate: `export const closeGate = internalMutation({ args: { scopeKey: v.string(), reason: v.string(), actor: actorValidator }, returns: v.null(), handler })` the operator's entry to `closeScope` for a tenant, a source stream type or the `all` scope, with no generation, which is also how a meaning migration's sweep is run under a pause when the operator wants one; the source scopes of a paused rebuild are closed by `startGeneration` and never ahead of it, because a closed entry can only admit a generation that exists (E-8, E-25, D9, D19) -- fnResumeScope: `resumeScope(ctx: MutationCtx, scopeKey: ScopeKey, actor: Actor): Promise` removing the scope from `closed` and writing the audit record in the caller's mutation; called by `resumeGate` and by the rebuild's `abortGeneration` (E-8, D9, D19) -- fnResumeGate: `export const resumeGate = internalMutation({ args: { scopeKey: v.string(), actor: actorValidator, abort: v.optional(v.boolean()) }, returns: v.null(), handler })` the operator's entry to `resumeScope`, refusing while the served generation is still building, verifying or verified unless `abort` is set, in which case it aborts the generation in the same mutation (E-8, D9, Sc L2-6) -- fnGetGate: `export const getGate = internalQuery({ args: {}, returns: gateDoc, handler })` for the operator and the acceptance tests (E-8, D19) -- occSerialization: a writer's mutation reads the gate document; a close that commits first invalidates that read, the writer re-runs, reads the closed gate and refuses; a writer that commits first wrote before the close, which is the consistent cut; a batch scheduled before the close but started after it reads the closed gate like any writer, so no closing state, drain interval or confirmation is needed, and the one-second mutation timeout plays no part (F1, D9, E-8) +- validatorClosedEntry: `const closedEntryValidator = v.object({ scopeKey: v.string(), reason: v.string(), generationId: v.optional(v.id("generations")), changedAt: v.number(), changedBy: v.string() })` where `changedBy` is the stated operator and `changedAt` the closing mutation's `Date.now()` (E-8, E-48) +- tableMaintenanceGates: `maintenanceGates: defineTable({ key: v.literal("gates"), closed: v.array(closedEntryValidator), updatedAt: v.number() }).index("by_key", ["key"])` holding at most one document, whose `closed` lists the entries in the order they were closed (E-8, D9) +- indexGatesByKeyUse: the one point read every writer makes, `.withIndex("by_key", (q) => q.eq("key", "gates")).unique()`; an absent document reads as no closed scope (E-8, D9) +- typeScopeKey: `type ScopeKey = "all" | string` where a tenant scope is `"tenant:" + tenantId` and a source scope is `"source:" + contextId + ":" + streamType`, built only by `scopesOfUseCase`, `startGeneration`, the rebuild's batches, the journal's baseline driver and the operator entries (E-8, E-38, D11) +- fnGateAllows: `gateAllows(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[], allowGeneration?: Id<"generations">): Promise<{ allowed: true } | { allowed: false; scopeKey: ScopeKey; reason: string }>` the one gate read: it first reads `process.env.MAINTENANCE_MODE` and answers `{ allowed: false, scopeKey: "all", reason: "restore" }` without reading the document when the value is `restore`; otherwise it reads the gate document once and answers not allowed with the `scopeKey` and `reason` of the first entry of `closed`, in array order, whose `scopeKey` is one of `scopes` and which is not exempt, and allowed when there is none; an entry is exempt only when `allowGeneration` is given, the entry has a `generationId`, and the two are equal; it writes nothing (E-8, E-42, D9, F9) +- fnAssertWritable: `assertWritable(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[]): Promise` calling `gateAllows` with no generation and, when the answer is not allowed, throwing through the boundary's `refuseTransient({ code: "writePaused", message: "write paused for " + scopeKey + ": " + reason })`; the command pipeline's step 7 calls it before it mints the operation, and nothing else does (E-8, D9, E-35) +- errorCodeWritePaused: `writePaused`, the transient code the outcome boundary reserves; the refusal's data is exactly `{ kind: "transient", code: "writePaused", message }` with no `retryAfterMs`, where `message` is `"write paused for all: restore"` under the restore door and `"write paused for tenant:" + tenantId + ": " + reason` for a closed tenant; a transient refusal, never a stored rejection, so no receipt, event, read-model row or audit record exists for the refused command (D4, Law 6, E-35) +- restoreDoor: `gateAllows` runs in the parent and reads `process.env.MAINTENANCE_MODE` before the gate document; only the exact value `restore` closes the door, so the restore procedure keeps writers out even though the snapshot replaced the gate, and every rebuild batch parks until the accepted branch resumes it; the restore procedure's own six mutations call neither `gateAllows` nor `assertWritable`; no component reads the variable (E-42, D19, F11) +- fnScopesOfUseCase: `scopesOfUseCase(tenantId: string, writes: readonly SourceRef[]): ScopeKey[]` returning `"all"`, the tenant scope and one source scope per `SourceRef` of the declaration's `writes`, in that order, so a command is held by a closed deployment, by its closed tenant and by a paused rebuild of any view whose sources it writes (E-8, E-38, D9) +- fnCloseScope: `closeScope(ctx: MutationCtx, scopeKey: ScopeKey, reason: string, operator: string, generationId?: Id<"generations">): Promise` throwing `"The scope " + scopeKey + " is already closed"` when `closed` holds an entry with that `scopeKey` and `"The gate holds 16 closed scopes, its limit"` when it holds `limitClosedScopes` entries; otherwise appending the entry to `closed`, inserting the document when there is none, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { action: "gate.close", scopeKey, reason, generationId, operator })` in the caller's mutation; called by `closeGate` and by the rebuild's `startGeneration`, which passes the ID of the generation row it has just inserted (E-8, E-43, E-48, D9, D19) +- fnResumeScope: `resumeScope(ctx: MutationCtx, scopeKey: ScopeKey, operator: string): Promise` throwing `"The scope " + scopeKey + " is not closed"` when `closed` holds no entry with that `scopeKey`; otherwise removing the entry, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { action: "gate.resume", scopeKey, reason: entry.reason, generationId: entry.generationId, operator })` in the caller's mutation, the reason and the generation being those of the entry it removed; called by `resumeGate` and by the rebuild's `abortGeneration` and `switchGeneration` (E-8, E-43, E-48, D9, D19) +- fnCloseGate: `export const closeGate = internalMutation({ args: { scopeKey: v.string(), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `closeScope` with no generation: it calls `assertOperator`, throws `"closeGate takes the scope all or a tenant scope, and " + scopeKey + " is neither"` unless `scopeKey` is `all` or `tenant:` followed by a tenant ID of 1 to `limitIdLength` bytes, throws `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and then calls `closeScope`; it does not read the tenant list, so a tenant may be closed before its first grant; this is also how a meaning migration's sweep is run under a pause (E-8, E-25, E-48, D9, D19) +- fnResumeGate: `export const resumeGate = internalMutation({ args: { scopeKey: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `resumeScope`: it calls `assertOperator`, makes the scope check of `closeGate` with `resumeGate` in the message, so a source scope is never reopened ahead of its rebuild, and then calls `resumeScope` (E-8, E-48, D9, Sc L2-6) +- fnGetGate: `export const getGate = internalQuery({ args: {}, returns: v.object({ restore: v.boolean(), closed: v.array(closedEntryValidator) }), handler })` answering `restore` true when `MAINTENANCE_MODE` is `restore` and `closed` as the document holds it, empty when there is no document (E-8, E-42, D19) +- fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page (E-8, E-43, D19) +- occSerialization: a writer's mutation reads the gate document, or the index range that holds none; a close that commits first invalidates that read, the writer re-runs, reads the closed gate and refuses; a writer that commits first wrote before the close, which is the consistent cut; a batch scheduled before the close but started after it reads the closed gate like any writer, so no closing state, drain interval or confirmation is needed, and the one-second mutation timeout plays no part (F1, D9, E-8) - limitClosedScopes: at most 16 scopes closed at once, so the gate document stays a few kilobytes and the writer check scans an array in memory (E-8, F13) -- limitGateReadsPerCommand: 1 point read per command; the first experiment counts it (E-8, Sc L2-3) +- limitGateReason: a reason is 1 to 256 bytes of UTF-8, so the refusal's message stays under a kilobyte (E-8, F13) +- limitGateReadsPerCommand: 1 point read per command, and none while the restore door is closed; the first experiment counts it (E-8, Sc L2-3) ## Verification — reviewed -- A reviewer confirms that `assertWritable` is called by the command pipeline's step 7 for every command, that `gateAllows` is called by every background batch before its first write and that the batch parks rather than throws on a refusal, that the two are one read because `assertWritable` is `gateAllows` plus the throw, that the restore procedure's six mutations are the only exemption and write only `restoreRuns` and the generation rows they resume, and that no function caches the gate across mutations. -- A reviewer confirms that no function other than `startGeneration` closes a source scope for a rebuild, so every closed entry that serves a rebuild names a generation row that exists, and that the journal's baseline driver reads the gate with the source scope of the stream type it migrates before every batch. +- A reviewer confirms that `assertWritable` is called by the command pipeline's step 7 for every command, that `gateAllows` is called by every background batch before its first write and that the batch parks rather than throws on a refusal, that the two are one read because `assertWritable` is `gateAllows` plus the throw, that the gate's entries and the restore procedure's six mutations are the only exemptions, and that no function caches the gate across mutations. +- A reviewer confirms that `gateAllows` reads the environment before the document, that an entry with no generation is never exempt, and that a gate read with no generation passes no closed entry. +- A reviewer confirms that no function other than `startGeneration` closes a source scope, so every closed entry that names a generation names a row that exists, and that the journal's baseline driver reads the gate with the source scope of the stream type it migrates before every batch. +- A reviewer confirms that `closeScope` and `resumeScope` are the only writers of the gate document, that each inserts its audit record in the same mutation with no catch around it, and that a test whose audit table refuses the record finds the gate unchanged. - A reviewer confirms that `writePaused` appears in the outcome boundary's closed code list as a transient refusal and is never stored as a rejection. -- A reviewer confirms that the restore workflow sets `MAINTENANCE_MODE` before the snapshot is restored and clears it only after its checks, and that `assertWritable` reads the variable before the gate document. +- A reviewer confirms that the restore workflow sets `MAINTENANCE_MODE` before the snapshot is restored and clears it only after its checks. diff --git a/design/specs/command/actor-and-scope.sdp.md b/design/specs/command/actor-and-scope.sdp.md index b244262..f3a9c20 100644 --- a/design/specs/command/actor-and-scope.sdp.md +++ b/design/specs/command/actor-and-scope.sdp.md @@ -16,7 +16,7 @@ relations: --- # Actor, tenant scope, namespace and grants -Layer 1 · Detail: full · Traces: D11, D2, D6, D13, Law 11, F11, E-6, E-54. +Layer 1 · Detail: full · Traces: D11, D2, D6, D13, Law 11, F11, E-6, E-54, E-48, E-49. This contract pins the shapes the parent establishes and passes into components: the `Actor` with its kind, identity and delegation reference, the `TenantScope`, the `CallerNamespace` and the `Authority` a worker runs under. It pins the `authorize` helper, the `authorizeQuery` helper a parent query calls, and the `grants` table. The event envelope's `actor` field, the receipts table's `namespace` field and the obligation record's `authority` field use these validators, so an actor and an authority are each defined once and referenced everywhere. @@ -31,6 +31,8 @@ This contract pins the shapes the parent establishes and passes into components: - [non-blocking] Extension E-6: the doc names the actor kinds, the envelope's actor field and the worker's two modes; the field names, the namespace union, the grant fields and the `authorize` signature are the design's, and the grant's `permission` string vocabulary is per product; the worker's two modes are one `Authority` shape, `{ actor, scope, mode }` with the literals `recheckDelegator` and `serviceAuthority`, stored on the obligation record and never restated, where the delegation travels on the actor's own `onBehalfOf` and `delegationRef`; and the doc says a derived command carries a server namespace without naming it, so this design assigns `worker` to every command a reaction issues, which is the namespace Sc L1-7 refuses to a client and the one D6 lists among the receipted callers, and reserves `system` for the platform's own maintenance chains and operator operations that run a receipted command, such as a restore or a migration driver, so a `system` receipt is never a reaction's; the owner confirms both (E-6, E-54, D6, D11, D13, Sc L1-7) - [non-blocking] Extension E-6: the tenant scope of an operation, of a command and of a query is the `tenantId` argument alone, so none takes a separate scope argument; this is a provisional reading the owner rules (E-6, Law 11, D11) - [non-blocking] Extension E-6: the doc says a query authorizes before it discloses and does not say how; here a parent query calls `authorizeQuery`, which establishes and authorizes its actor through the same two helpers a command's entry uses and is generic over the parent's data model; the owner has not ruled (E-6, D11, Law 5) +- [non-blocking] Extension E-48, the operator entry: the doc lists operators among the actors and names operator duties without saying how an operator reaches them; the option taken here is that admin access is the authority of an operation over the whole deployment, so every operator entry is an internal function that only admin access reaches, takes the operator as a required string, reads no grant and records that string on what it changes; a stated operator is what admin access chose to state and proves no identity, D11's one trusted deployment is the boundary, and no scope beyond the tenant is added (E-48, D11, D19) +- [non-blocking] Extension E-49, the tenant list: the doc has every record name its tenant and names no list of tenants, while a batch over the whole deployment needs the next tenant; the option taken here is a `tenants` table in the parent with one row per tenant, written by `insertGrant` in the mutation that inserts a tenant's first grant and never deleted, read one tenant at a time in tenant-ID order; the do-nothing option, reading the distinct tenant IDs off the `grants` index, is not taken because revocation deletes grant rows and a tenant whose grants are all revoked still owns records (E-49, D11, Law 11, E-37) ## Contract @@ -44,6 +46,12 @@ This contract pins the shapes the parent establishes and passes into components: - `authorize` reads the grants for the actor in the tenant and answers allowed or denied with a reason; it never executes, discloses or caches (Law 5, D11) - A context operation receives `tenantId`, `actor` and `operation`, the `OperationRef` of `spec:context.event-envelope`, as arguments on every call, a context query receives `tenantId`, and neither reads `ctx.auth` (D11, F11) - The `actorValidator` here is the one the event envelope stores in its `actor` field and the receipts table stores as `actorId`, so an actor is defined once (D2, D6) +- [extension] An operator entry is an `internalMutation` or `internalQuery`, so only admin access reaches it; it reads no grant, calls neither `establishActor` nor `authorize`, and reads no `ctx.auth` (E-48, D11, D19) +- [extension] An operator entry that changes anything takes `operator`, a required string of 1 to 512 bytes of UTF-8 that is not only whitespace, checks it with `assertOperator` before any read or write, and records it unchanged on what it changes: the generation row, the gate's closed entry and its audit record, or the restore's run record; an operator entry that only reads takes none (E-48, D19) +- [extension] A stated operator is what admin access chose to state: no function establishes an actor of kind `operator` from a credential, no permission names an operator duty, and the kind `operator` of `ActorKind` is carried only by an actor a trusted internal caller states (E-48, D11) +- [extension] The tenant list is the `tenants` table: `insertGrant` inserts a tenant's row when it finds none, in the same mutation as the grant, so every tenant that a command or a query was ever authorized in has a row; the library never deletes a row, and revoking every grant of a tenant leaves its row (E-49, D11, Law 11) +- [extension] Grant rows are inserted only through `insertGrant`, so no tenant can hold a grant without a row in the tenant list (E-49, E-37, D11) +- [extension] A pass over the whole deployment, a rebuild's batches, a restore's checks and a receipt sweep among them, takes its tenants from `nextTenant`, one tenant per read in ascending tenant-ID order, and names that tenant on every read and write of tenant data; a tenant whose row is inserted behind a pass's cursor is not visited by that pass (E-49, Law 11, D11) ## Design @@ -65,9 +73,9 @@ This contract pins the shapes the parent establishes and passes into components: - fnEstablishActor: `establishActor(ctx: MutationCtx | QueryCtx, serviceIssuers: ReadonlySet): Promise` returns `null` when `ctx.auth.getUserIdentity()` is `null`, a `service` actor when the identity's issuer is in `serviceIssuers`, and a `human` actor otherwise, with `id` from `tokenIdentifier` (D11, E-37) - typeQueryPolicy: `type QueryPolicy = { name: string; tenantId: string; permission: string; subject?: SubjectRef }` what a parent query states before it discloses anything: its own name, the tenant it reads and the permission the read requires (D11, Law 5, E-6) - fnAuthorizeQuery: `authorizeQuery(ctx: GenericQueryCtx, policy: QueryPolicy, serviceIssuers?: ReadonlySet): Promise` calls `establishActor` and then `authorize` for the policy's tenant, permission and subject, as the pipeline's steps 2 and 4 do, and returns the actor; it throws through `reject` with code `unauthenticated` when there is no identity and `forbidden` with `details.reason` when the decision denies, each with the query's `name` where a command's type goes; it writes nothing, and it is generic over the parent's data model because a database reader typed with the parent's tables is not assignable to one typed with the library's (D11, Law 5, D7, E-6) -- fnInsertGrant: `insertGrant(ctx: { db: MutationCtx["db"] }, grant: GrantInput): Promise>` inserts one grant row with `grantedAt` the mutation's `Date.now()`, where `GrantInput = { tenantId: string; principalKind: ActorKind; principalId: string; permission: string; subject?: SubjectRef; grantedBy: string }`; a plain helper the library exports and registers no function for (D11, E-37) +- fnInsertGrant: `insertGrant(ctx: { db: MutationCtx["db"] }, grant: GrantInput): Promise>` reads `tenants` by `by_tenant` for `grant.tenantId` and inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, then inserts one grant row with `grantedAt` the mutation's `Date.now()`, where `GrantInput = { tenantId: string; principalKind: ActorKind; principalId: string; permission: string; subject?: SubjectRef; grantedBy: string }`; a plain helper the library exports and registers no function for (D11, E-37, E-49) - fnRevokeGrant: `revokeGrant(ctx: { db: MutationCtx["db"] }, grant: Omit): Promise` deletes the rows that grant this permission to this principal, on the named subject when one is given and otherwise on any, and returns how many it deleted; revocation is that delete (D11, E-37) -- grantWriters: the application registers its own internal mutations over `insertGrant` and `revokeGrant`; a tenant's first grant is created by an operator who runs that internal mutation with admin access, and a native test does the same as setup with the harness's admin access (D11, E-37, E-15) +- grantWriters: the application registers its own internal mutations over `insertGrant` and `revokeGrant`; a tenant's first grant is created with admin access through that internal mutation, and later grants the same way or by whatever command the composition registers; the library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation, which are an adopter's to build on the helpers; a native test creates its grants as setup with the harness's admin access (D11, E-37, E-15) - tableGrants: `grants: defineTable({ tenantId: v.string(), principalKind: actorKindValidator, principalId: v.string(), permission: v.string(), subject: v.optional(v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() })), grantedBy: v.string(), grantedAt: v.number() }).index("by_principal", ["tenantId", "principalKind", "principalId"]).index("by_permission", ["tenantId", "permission"])` (D11, Law 11, E-6) - indexGrantsByPrincipal: `.index("by_principal", ["tenantId", "principalKind", "principalId"])` (D11) - indexGrantsByPrincipalUse: the authorization read of every command and disclosing query, three equalities and `.take(limit + 1)` against the bound of `limitGrantsRead` (D11, Law 5) @@ -75,6 +83,13 @@ This contract pins the shapes the parent establishes and passes into components: - indexGrantsByPermissionUse: operator inspection of who holds a permission in a tenant; never on the command path (D19, E-6) - accessContextOption: when grants need an audit journal, a dedicated access context owns them and `authorize` reads `ctx.runQuery(components.access.grants.forPrincipal, input)` inside the same transaction; the decision type is unchanged (D2, D11) - limitGrantsRead: at most 500 grants per authorization read; the read takes 501 rows and throws a plain `Error`, a technical failure, when 501 come back, because `.take(n)` alone returns at most `n` rows and never shows whether more exist, so a 501st grant is surfaced and never silently truncated (F13, E-37) +- tableTenants: `tenants: defineTable({ tenantId: v.string(), createdAt: v.number() }).index("by_tenant", ["tenantId"])` in `commandTables`, the schema fragment that holds `grants`; one row per tenant, at most `limitIdLength` bytes of tenant ID (E-49, D11, Law 11) +- indexTenantsByTenantUse: the point read of `insertGrant` and the range read of `nextTenant`; no command and no query reads the table (E-49) +- fnNextTenant: `nextTenant(ctx: { db: QueryCtx["db"] }, after: string | null): Promise` making one indexed read, `ctx.db.query("tenants").withIndex("by_tenant", (q) => (after === null ? q : q.gt("tenantId", after))).first()`, and answering that row's `tenantId`, or `null` when no tenant follows `after`; `null` as `after` asks for the first tenant (E-49, Law 11) +- validatorOperator: `const operatorValidator = v.string()` the `operator` argument of every operator entry that changes anything and the type of every field that records it (E-48) +- fnAssertOperator: `assertOperator(operator: string): string` returning its argument, and throwing the plain `Error` `"An operator entry needs a stated operator"` when `operator.trim()` is empty and `"The stated operator is " + bytes + " bytes, above the limit of 512"` when `utf8Length(operator)` is above `limitOperatorBytes`; an operator entry calls it first, so a refused entry has read and written nothing (E-48, D19) +- limitOperatorBytes: 512 bytes of UTF-8, the bound `limitCallText` of `spec:command.command-pipeline` gives an actor's `id` (E-48, E-34) +- operatorRecordedWhere: `startedBy` and `changedBy` on the generation row of `spec:application.generation-registry`, `changedBy` on a closed entry of the gate and `operator` on its audit record in `spec:application.write-pause` and `spec:operations.baseline-operations`, and `operator` on the run record of `spec:application.restore` (E-48, D19) ## Verification — reviewed diff --git a/design/specs/command/command-pipeline.sdp.md b/design/specs/command/command-pipeline.sdp.md index c7ef51b..070cd6c 100644 --- a/design/specs/command/command-pipeline.sdp.md +++ b/design/specs/command/command-pipeline.sdp.md @@ -84,7 +84,7 @@ The pipeline is one code path with two registered entries per command. The publi - Step 5, receipt lookup: for a receipted call the pipeline builds the key from tenant, namespace, command type and request key and reads the `receipts` table by its key index in the same transaction; no receipt means new intent, a receipt whose `contractVersion` differs from the declaration's throws rejection `unsupportedContractVersion` before any fingerprint is compared, a receipt with the same fingerprint means a duplicate and the pipeline returns the stored thin outcome with `replayed` true and `result` null, a receipt with a different fingerprint throws rejection `idempotencyConflict`, and neither rejection carries details or anything else of the stored receipt except the one fact the two codes cannot hide, whether the receipt's contract version is the declaration's (D6, Law 4, Law 5, Sc L1-3, Sc L1-5, E-5) - Step 5, disclosure: a duplicate's stored outcome is returned only because step 4 ran for this call, so a caller whose authorization was revoked after the original run receives `forbidden` and learns nothing (D6, Law 5, Sc L1-8) - Step 6, admit: for new intent only, the pipeline evaluates the declaration's admission policy; a refusal throws transient `rateLimited` or `capacity`, nothing is stored, and the same intent may be retried (D6, D4, Sc L1-9) -- Step 7, gate and mint the operation: the pipeline reads the maintenance gate once through `assertWritable(ctx, scopesOfUseCase(tenantId, (decl.readModels ?? []).map((r) => r.readModel.name), decl.writes))` of `spec:application.write-pause`, which refuses with the transient `writePaused` while a scope of this command is closed, the tenant, a read model it maintains or a stream type it writes, or the deployment is in restore, then mints the whole `OperationRef`, the operation ID, `causedBy` from the call's `causedBy` argument when the internal entry received one and otherwise of kind `command` with the declaration's name, and the caller's optional correlation ID, and passes it to every context call so that events in different streams relate through it (Vocab: operation, D2, D9, D13, E-8, E-26, E-31) +- Step 7, gate and mint the operation: the pipeline reads the maintenance gate once through `assertWritable(ctx, scopesOfUseCase(tenantId, decl.writes))` of `spec:application.write-pause`, which refuses with the transient `writePaused` while a scope of this command is closed, the whole deployment, the tenant or a stream type it writes, or the deployment is in restore, and only then mints the whole `OperationRef`, the operation ID, `causedBy` from the call's `causedBy` argument when the internal entry received one and otherwise of kind `command` with the declaration's name, and the caller's optional correlation ID, and passes it to every context call so that events in different streams relate through it (Vocab: operation, D2, D9, D13, E-8, E-26, E-31) - Step 8, execute: the executor captures the outside facts the decision needs, from parent tables or configuration read in this transaction, and calls each context named by the declaration once through `ctx.runMutation` on the component's sanctioned operation, `components..operations.`, with tenant scope, actor, the operation reference, the command's input as lists and the captured facts; each call is an isolated sub-transaction that commits with the parent and rolls back on throw; a rejection thrown inside a context rolls its sub-transaction back and propagates to the boundary's outermost rethrow, and the executor returns the kernel's `CommittedOutcome` together with the `streams` entries every context call returned, never a rejection value (F2, F3, D3, D10, D7, E-23) - Step 9, update read models: the pipeline first checks that every `streams` entry the executor returned names a context and stream type in the declaration's `writes`, throwing a plain error, a defect and a technical failure, when one does not, so the list the gate of step 7 was derived from is proven on the first native run; then, for every binding, it reads the generations to write through `generationsToWrite`, throws a plain error that names the command and the read model when there is none, and calls `applyProjection` of `spec:application.projection-contract` for each matching entry with the entry's DTO and version and the list unchanged, in the live-created mode when the entry's `created` flag is set and the live-updated mode otherwise, or `applyAggregate` or `applyHistoryProjection` for a binding of those forms, in the same transaction; it counts the rows those calls insert, replace or delete, a row in each generation once, and after each entry is applied throws a plain error when they pass `limitReadModelWritesPerCommand`; the executor writes no read-model table itself (D8, D9, D12, Law 9, E-7, E-8, E-9, E-23, E-45) - Step 10, insert the receipt and the audit record: for a receipted call the pipeline inserts the thin receipt, outcome kind, operation ID, affected IDs derived from the stream versions, and the stream versions, in the same transaction as the lookup of step 5, so two concurrent identical requests serialize under optimistic concurrency and only one effect commits; for a declaration that sets `audit` it then calls `writeAudit` of `spec:operations.baseline-operations`, whose failure throws and rolls the command back (D6, D19, F1, Sc L1-3, E-43) @@ -109,7 +109,7 @@ The pipeline depends on three contracts it does not own: the context component's - validatorCommandResponse: `const commandResponseValidator = (output: Validator) => v.union(v.object({ kind: outcomeKindValidator, result: output, operationId: v.string(), affected: v.array(affectedRefValidator), versions: v.array(streamVersionValidator), replayed: v.literal(false) }), v.object({ kind: outcomeKindValidator, result: v.null(), operationId: v.string(), affected: v.array(affectedRefValidator), versions: v.array(streamVersionValidator), replayed: v.literal(true) }))` where `outcomeKindValidator = v.union(v.literal("applied"), v.literal("businessFailure"))`, the composition helper passes the declaration's `output`, and `streamVersionValidator` is the kernel's; Convex validates the return value and would throw on a replay if the executed member alone were declared, which is why the replayed member is part of the closed type (D4, D12, D6, E-30) - operationRef: `operation = { operationId: crypto.randomUUID(), causedBy: call.causedBy ?? { kind: "command", commandType: decl.name }, ...(call.correlationId === undefined ? {} : { correlationId: call.correlationId }) }` minted once per top-level mutation at step 7, before the first context call, passed to the executor and by it to every context call unchanged, stored on every event and its ID on the receipt; a duplicate returns the receipt's `operationId`; a reaction passes `causedBy: { kind: "event", tenantId, contextId, eventId }` naming its obligation's source event, as `spec:obligations.fan-out-and-chains` pins, and that is the one producer of the envelope's `event` variant (Vocab: operation, D2, D13, E-26, E-31, E-54) - ctxAuthUse: `ctx.auth.getUserIdentity()` is called once, in the public entry at step 2, and its `tokenIdentifier`, `subject` and `issuer` become the actor's identity; no component function reads `ctx.auth`, by this design's rule (D11, F11) -- gateRead: `assertWritable` of `spec:application.write-pause` runs at step 7 with the scopes `scopesOfUseCase` derives from the tenant, the declaration's read models and the declaration's `writes`, the context and stream type pairs the executor may write, one point read per command after the environment's restore door; a refusal is the transient `writePaused` and nothing is stored (D9, D4, E-8, E-35, E-38, E-42) +- gateRead: `assertWritable` of `spec:application.write-pause` runs at step 7 with the scopes `scopesOfUseCase` derives from the tenant and the declaration's `writes`, the context and stream type pairs the executor may write, one point read per command after the environment's restore door; a refusal is the transient `writePaused` and nothing is stored (D9, D4, E-8, E-35, E-38, E-42) - factsCapture: the executor is the one place that captures outside facts, such as an accepted price list, a currency or a policy version, from parent tables or configuration inside this transaction, and passes them as `facts` on each context call; they reach the decider through its decision context, never through `input`, so the fingerprint of step 5 excludes them (D3, E-22, E-36) - contextCall: `await ctx.runMutation(components.inventory.operations.allocate, { tenantId, actor, operation, input: { orderId, lines }, facts })`, one call per context named by the declaration, lists in and the operation outcome out, the DTO with the versions and one `streams` entry per planned stream with its DTO, version, created flag and the events the call appended; the `args` shape is the one `spec:context.context-component` fixes, `operation` is the `OperationRef` of `spec:context.event-envelope` minted at step 7, and the return is the `OperationOutcome` of `spec:context.persistence-adapter` (D10, F2, E-22, E-23, E-26) - contextCallIsolation: each component call is a sub-transaction isolated from the other calls; a throw inside it rolls that sub-transaction back, and uncaught it rolls the parent back, so the parent never observes a half-applied context (F2, F3, S2) diff --git a/design/specs/command/tenancy-and-authority.sdp.md b/design/specs/command/tenancy-and-authority.sdp.md index 2171f4c..78a743b 100644 --- a/design/specs/command/tenancy-and-authority.sdp.md +++ b/design/specs/command/tenancy-and-authority.sdp.md @@ -36,7 +36,7 @@ This Spec owns the rules of tenant scope, actor establishment, authorization and - [non-blocking] Extension E-37: the doc says the parent authenticates and that grants are authoritative data, but not how an identity becomes an actor or how revocation takes effect; here the public entry and a parent query's `authorizeQuery` map `ctx.auth.getUserIdentity()` to a human actor, or a service actor when the issuer is a configured service issuer, grants live in a parent table by default and are revoked by deleting the row, with no cache, so a revoked grant fails the next command (E-37, D11) - [non-blocking] Extension E-6: the doc names the actor kinds and the worker's two modes but not the types; the actor, scope, namespace, grant and `Authority` shapes are the design's and are pinned once in `spec:command.actor-and-scope`, which also rules that a derived command's namespace is `worker` (E-6, D11) -- [non-blocking] Extension E-37: the design takes grants as written by plain library helpers, `insertGrant` and `revokeGrant`, over which the application registers its own internal mutations, with a tenant's first grant created by an operator who runs that internal mutation with admin access; who creates the first grant and who may change grants is a provisional reading the owner rules (E-37, D11, Law 5) +- [non-blocking] Extension E-37: the design takes grants as written by plain library helpers, `insertGrant` and `revokeGrant`, over which the application registers its own internal mutations, with a tenant's first grant created with admin access through the composition's own internal grant mutation, and later grants the same way or by whatever command the composition registers; the library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation, which are an adopter's to build on the helpers (E-37, D11, Law 5) ## Behavior diff --git a/design/specs/operations/baseline-operations.sdp.md b/design/specs/operations/baseline-operations.sdp.md index f6679f7..277519d 100644 --- a/design/specs/operations/baseline-operations.sdp.md +++ b/design/specs/operations/baseline-operations.sdp.md @@ -21,7 +21,7 @@ relations: --- # Baseline operations -Layer 1 to 2 · Detail: full · Traces: Thesis, D11, D13, D18, D19, Law 8, Law 11, F1, F12, F13, Sc ALL-1, Sc L2-8, E-43. +Layer 1 to 2 · Detail: full · Traces: Thesis, D11, D13, D18, D19, Law 8, Law 11, F1, F12, F13, Sc ALL-1, Sc L2-8, E-43, E-48. Operations travel with the capability. This Spec carries the part of D19 that applies from Layer 1: diagnostics that never cancel a valid write and audit that fails closed, bounds on every batch, small events with personal data behind references, a retention and deletion policy before production data, diagnosis by identifiers with stored receipts as the authority, the metric list, and a release that preserves, migrates or drains pending work. The restore procedure of the transactional profile is `spec:application.restore`. What applies to obligations, external dispatch and provider evidence is carried by the obligation module in Layer 3. @@ -39,7 +39,7 @@ Security, bounded resource use, useful errors and tests start in Layer 1, so non ### Open questions -- [non-blocking] Extension E-43: the doc says audit is written inside the transaction and fails closed, that diagnostics hold no raw prompts, credentials or unbounded payloads, and that personal data sits behind references; it does not give the audit record, the diagnostic record or the reference. The option taken here: an `auditRecords` table with indexes by operation, subject and request key, a bounded diagnostic record emitted through a sink that never throws, and a `personalData` table that events point at by reference (E-43, D19) +- [non-blocking] Extension E-43: the doc says audit is written inside the transaction and fails closed, that diagnostics hold no raw prompts, credentials or unbounded payloads, and that personal data sits behind references; it does not give the audit record, the diagnostic record or the reference. The option taken here: an `auditRecords` table with indexes by operation, subject and request key, whose every field has a stated source in the command, an `operatorAudit` table for the gate changes an operator entry makes, which name no tenant when the scope is the whole deployment, a bounded diagnostic record emitted through a sink that never throws, and a `personalData` table that events point at by reference. The do-nothing option is Convex's own audit logging, which the pinned `convex` package declares as `audit(body)`; it is not taken as the record, because an entry in a log outside the database is not read by the operator's queries by operation, subject and request key, and whether such an entry proves a commit is not shown by the pinned package; a deployment may add it beside the tables as an export (E-43, D19) - [non-blocking] The retention and deletion policy's content is the owner's: which historical claims survive deletion and how long receipts, audit records and diagnostics are kept; the design gives the hooks and states no periods (D19) ## Rule @@ -61,9 +61,10 @@ Security, bounded resource use, useful errors and tests start in Layer 1, so non - Accepted deferred work stores a logical handler key and version, stable shims stay while vendor jobs refer to old function handles, and an unsupported version goes to needs attention, never an endless retry; this applies from Layer 3 and its detail is the obligation module's (D19, D13) - Security, bounded resource use, useful errors and tests start in Layer 1 (Thesis) - Every audit record and every diagnostic record names its tenant, and every operator query over them takes the tenant scope (Law 11, D11) -- Operator functions that read audit or diagnostics are internal functions reached through an operator actor; a client cannot read another tenant's audit (D11, Law 5) +- Operator entries that read audit records are internal functions, so only admin access reaches them, as the operator entry contract of `spec:command.actor-and-scope` rules; each one over tenant data takes the tenant, so a client cannot read another tenant's audit (D11, Law 5, E-48) - [extension] A diagnostic record is emitted per command through a sink that never throws, is bounded to 4 KiB, and holds identifiers, outcome kind, error code, versions and timings only (E-43, D19) -- [extension] An audit record is one document written by the command pipeline's step 10 through `writeAudit` for every declaration whose `audit` field is set, and the failure of that write throws and rolls back the command (E-43, D19, E-38) +- [extension] An audit record is one document written by the command pipeline's step 10 through `writeAudit` for every declaration whose `audit` field is set, after an applied command and after a business failure alike, with every field taken from the command as `auditRecordSources` lists, and the failure of that write throws and rolls back the command (E-43, D19, E-38) +- [extension] An operator entry that changes the maintenance gate writes one `operatorAudit` record through `writeOperatorAudit` in the same mutation, with the stated operator, and the failure of that write throws and rolls the change back; the generation row and the restore's run record carry their operator themselves and write no audit record (E-43, E-48, E-8, D19) - [extension] Personal data that rebuild does not need lives in the `personalData` table and is referenced from events by a stable reference; deleting the row leaves the event's claim intact and its personal fields unreadable (E-43, D19) ## Design @@ -73,18 +74,23 @@ Diagnostics and audit are two paths with opposite failure rules. `emitDiagnostic Optimistic-concurrency retries cannot be observed from inside a mutation, because a retried mutation re-runs from the start; the metric comes from the platform's function logs and insights. Rebuild and restore progress are read from the generation and restore run rows. The Layer 3 metrics are listed so the list is complete; they are measured once the obligation module is installed. - transactionBoundary: audit is written inside the command's top-level mutation and fails closed; diagnostics are emitted from the same mutation after the business writes and never throw (D19, Sc ALL-1) -- convexSurface: `getAuditByOperation`, `getAuditBySubject`, `getAuditByRequestKey` as internal queries for operators; the diagnostic sink is `console.log` of one JSON line, read through the platform's logs (E-43, D19) -- tableAuditRecords: `auditRecords: defineTable({ tenantId: v.string(), operationId: v.string(), requestKey: v.optional(v.string()), commandType: v.string(), actor: actorValidator, subject: v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() }), kind: v.union(v.literal("security"), v.literal("business")), decision: v.string(), causedBy: v.optional(v.string()), recordedAt: v.number(), details: v.optional(v.any()) })` (E-43, D19, Law 11) +- convexSurface: `getAuditByOperation`, `getAuditBySubject`, `getAuditByRequestKey` as internal queries for operators, each taking `tenantId` first, and `getGateAudit` of `spec:application.write-pause` over `operatorAudit`; the diagnostic sink is `console.log` of one JSON line, read through the platform's logs (E-43, D19) +- tableAuditRecords: `auditRecords: defineTable({ tenantId: v.string(), operationId: v.string(), requestKey: v.optional(v.string()), commandType: v.string(), actor: actorValidator, subject: v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() }), kind: v.union(v.literal("security"), v.literal("business")), decision: v.union(v.literal("applied"), v.literal("businessFailure")), causedBy: causedByValidator, recordedAt: v.number() })` in `auditTables`, the schema fragment the library exports, where `causedByValidator` is the envelope's (E-43, D19, Law 11) +- typeAuditRecordInput: `type AuditRecordInput = { tenantId: string; operationId: string; requestKey?: string; commandType: string; actor: Actor; subject: SubjectRef; kind: "security" | "business"; decision: "applied" | "businessFailure"; causedBy: CausedBy }`, the table's document without `recordedAt`, which `writeAudit` sets to the mutation's `Date.now()` (E-43, D19) +- auditRecordSources: step 10 builds the input from the command alone: `tenantId`, `requestKey` and `actor` from the `PipelineCall`, `operationId` and `causedBy` from the `OperationRef` step 7 minted, `commandType` from the declaration's `name`, `kind` from the declaration's `audit.kind`, `decision` from the outcome's kind, and `subject` from the declaration's `permission.subjectFrom(input)` when it is declared and otherwise from the first entry of the outcome's `versions`, its `contextId`, `streamType` and `streamId`; a declaration that sets `audit`, declares no `subjectFrom` and returns no version throws a plain error, so the command fails closed (E-43, E-38, D19) - indexAuditByOperation: `.index("by_operation", ["tenantId", "operationId"])` (E-43) -- indexAuditByOperationUse: diagnosis by operation ID and the restore's receipt check (D19, E-42) +- indexAuditByOperationUse: diagnosis by operation ID (D19) - indexAuditBySubject: `.index("by_subject", ["tenantId", "subject.contextId", "subject.streamType", "subject.streamId", "recordedAt"])` (E-43) - indexAuditBySubjectUse: diagnosis by business subject, newest first (D19) - indexAuditByRequestKey: `.index("by_request_key", ["tenantId", "requestKey"])` (E-43) - indexAuditByRequestKeyUse: diagnosis by request key when the receipt has expired (D19, D6) - typeDiagnosticRecord: `[extension] interface DiagnosticRecord { tenantId: string; operationId: string; requestKey?: string; commandType: string; actorKind: string; actorId: string; outcome: "applied" | "businessFailure" | "rejection" | "technicalFailure" | "transientRefusal"; errorCode?: string; versions: StreamVersion[]; causedBy?: string; durationMs: number; layer: number }` (E-43, D19) - fnEmitDiagnostic: `emitDiagnostic(record: DiagnosticRecord, sink: DiagnosticSink = consoleSink): void` truncating to 4 KiB and swallowing every error the sink throws (E-43, D19, Sc ALL-1) -- fnWriteAudit: `writeAudit(ctx: MutationCtx, record: AuditRecordInput): Promise>` inserting with the validator and letting every failure propagate (E-43, D19, Sc ALL-1) -- faultInjection: `DiagnosticSink` and the audit table name are injectable in the fixture app only, so the acceptance test can break each subsystem; the production composition has no injection point (Sc ALL-1, Acceptance scenarios) +- fnWriteAudit: `writeAudit(ctx: { db: GenericDatabaseWriter }, record: AuditRecordInput): Promise>` inserting the record with `recordedAt` through `ctx.db.insert` and letting every failure propagate; `AuditDataModel` is the data model of `auditTables` (E-43, D19, Sc ALL-1) +- tableOperatorAudit: `operatorAudit: defineTable({ action: v.union(v.literal("gate.close"), v.literal("gate.resume")), scopeKey: v.string(), reason: v.string(), generationId: v.optional(v.id("generations")), operator: v.string(), recordedAt: v.number() }).index("by_scope", ["scopeKey", "recordedAt"])` in `auditTables`; operational data of the deployment like the generation registry, so its index leads with the scope and not with a tenant, the deviation from Law 11's index rule that the registry records under E-8 (E-43, E-48, E-8, D19) +- indexOperatorAuditByScopeUse: `getGateAudit`, who closed and who reopened one scope, newest first; no command reads the table (E-43, D19) +- fnWriteOperatorAudit: `writeOperatorAudit(ctx: { db: GenericDatabaseWriter }, record: OperatorAuditInput): Promise>` inserting the record with `recordedAt` at the mutation's `Date.now()` and letting every failure propagate, where `type OperatorAuditInput = { action: "gate.close" | "gate.resume"; scopeKey: string; reason: string; generationId?: Id<"generations">; operator: string }`; called by the write pause's `closeScope` and `resumeScope` and by nothing else (E-43, E-48, D19) +- faultInjection: `DiagnosticSink` and the audit table name are injectable in the fixture app only, so the acceptance test can break each subsystem; the production composition has no injection point; the gate's audit is broken without an injection point, by a `convex-test` schema whose `operatorAudit` table has a validator that refuses the record (Sc ALL-1, Acceptance scenarios, E-43) - tablePersonalData: `personalData: defineTable({ tenantId: v.string(), ref: v.string(), subject: v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() }), fields: v.any(), createdAt: v.number(), deletedAt: v.optional(v.number()) }).index("by_ref", ["tenantId", "ref"]).index("by_subject", ["tenantId", "subject.contextId", "subject.streamType", "subject.streamId"])` (E-43, D19, Law 11) - personalDataReference: an event payload that would hold personal data rebuild does not need holds `personalRef` instead; a deletion sets `deletedAt` and clears `fields`, and the retention policy states which claims the event still makes afterward (E-43, D19) - retentionHooks: receipts expire by the receipt contract, audit records and diagnostics by the retention policy, personal data by deletion requests; the design sets no period (D19, D6) @@ -100,7 +106,6 @@ Optimistic-concurrency retries cannot be observed from inside a mutation, becaus - metricRestoreProgress: checks passed, examined counts and findings per restore run, from the restore run record (D19, Sc L2-8) - releaseRule: a release keeps a building generation's cursor, a retired generation's rows and an import's progress row readable; a schema change to any of them ships with a migration that runs before the new code reads them (D19) - limitDiagnosticRecord: 4 KiB per diagnostic record; larger records are truncated, never dropped (E-43, D19) -- limitAuditDetails: 16 KiB per audit record's `details`; a larger record is a developer error that fails closed (E-43, D19, F13) - limitOperatorQuery: audit queries are paginated with at most 100 records per page (E-43, F13) ## Example space From 1129f1c64590c684e4cd2277739eb071660e2dda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:07:30 +0200 Subject: [PATCH 009/155] design: the twelve decisions of the forks recorded as the owner's, with the owner's words Co-Authored-By: Claude Opus 5.5 --- design/decisions/forks/OD-025.md | 2 +- design/decisions/forks/OD-046.md | 2 +- design/decisions/forks/OD-047.md | 2 +- design/decisions/forks/OD-064.md | 2 +- design/decisions/forks/OD-068.md | 2 +- design/decisions/forks/OD-069.md | 2 +- design/decisions/forks/OD-123.md | 2 +- design/decisions/forks/OD-148.md | 2 +- design/decisions/forks/OD-150.md | 2 +- design/decisions/forks/OD-151.md | 2 +- design/decisions/forks/OD-152.md | 2 +- design/decisions/forks/OD-153.md | 2 +- design/decisions/register.json | 96 ++++++++++++++++---------------- 13 files changed, 60 insertions(+), 60 deletions(-) diff --git a/design/decisions/forks/OD-025.md b/design/decisions/forks/OD-025.md index 3cb4017..ace9e54 100644 --- a/design/decisions/forks/OD-025.md +++ b/design/decisions/forks/OD-025.md @@ -52,7 +52,7 @@ The provisional reading is the composition's internal mutation run with admin ac The first grant of a tenant is created with admin access through the composition's own internal grant mutation, and later grants the same way or by whatever command the composition registers. The library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation: those are an adopter's to build on the helpers. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-046.md b/design/decisions/forks/OD-046.md index cc582e5..e3f06cf 100644 --- a/design/decisions/forks/OD-046.md +++ b/design/decisions/forks/OD-046.md @@ -49,7 +49,7 @@ The speed targets are set here, before the benchmark, as the doc requires. They - Contention, N commands at once on one stock item with stock for one: at N of 2 and 8 every command reaches a business outcome, one applied and the rest rejected for short stock, and the engine fails none after its bounded retries. At N of 32 the same count is reported, and a command the engine failed is a recorded cost and no failure of the experiment, because the caller repeats it under the same request key. - A target that is missed is reported as not held with its number. No target is changed after the run. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-047.md b/design/decisions/forks/OD-047.md index 5c03e72..d757709 100644 --- a/design/decisions/forks/OD-047.md +++ b/design/decisions/forks/OD-047.md @@ -43,7 +43,7 @@ The row has no provisional reading; the two references remain in `AGENTS.md`. [A The two references in `AGENTS.md` under "On the owner's machine" stay. They are paths and hold no content of the files. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-064.md b/design/decisions/forks/OD-064.md index 2b0b8b9..7ad79f2 100644 --- a/design/decisions/forks/OD-064.md +++ b/design/decisions/forks/OD-064.md @@ -42,7 +42,7 @@ The provisional reading measures at 100. [design/specs/application/first-experim One answer for the largest order: 100 lines is the largest order `PlaceOrder` of the example supports, and the bound its declaration carries. It is the example's promise and no platform limit: a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings. One document per stream stays the default mapping, with the derived mapping for state that outgrows its budget. The Specs stop calling 100 provisional. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-068.md b/design/decisions/forks/OD-068.md index 34ef25b..d741d91 100644 --- a/design/decisions/forks/OD-068.md +++ b/design/decisions/forks/OD-068.md @@ -42,7 +42,7 @@ The provisional reading keeps retired generations unwritten and reverifies rollb A retired generation is not written. Rollback reopens it as `verifying`, runs one verify pass that repairs what the intervening writes changed, and then switches. The switch is one write in both directions; the Specs stop promising a rollback of one write, and say rollback is a verify pass and then a switch of one write. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-069.md b/design/decisions/forks/OD-069.md index 479cd04..4c3a6e8 100644 --- a/design/decisions/forks/OD-069.md +++ b/design/decisions/forks/OD-069.md @@ -42,7 +42,7 @@ The provisional reading is deployment-wide generations with tenant-leading marke One generation per read model spans every tenant. The generation registry is operational data of the deployment and its indexes lead with the read model. Every table that holds tenant data, the read-model rows and the markers among them, still leads with the tenant. The deviation from Law 11's index rule is recorded where it is today, as confirmed. A rebuild for one tenant is not offered. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-123.md b/design/decisions/forks/OD-123.md index e454d60..98e69ca 100644 --- a/design/decisions/forks/OD-123.md +++ b/design/decisions/forks/OD-123.md @@ -52,7 +52,7 @@ The provisional reading is one document with the maximum as a placeholder. [desi One answer for the largest order: 100 lines is the largest order `PlaceOrder` of the example supports, and the bound its declaration carries. It is the example's promise and no platform limit: a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings. One document per stream stays the default mapping, with the derived mapping for state that outgrows its budget. The Specs stop calling 100 provisional. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-148.md b/design/decisions/forks/OD-148.md index e21d692..6fd89e0 100644 --- a/design/decisions/forks/OD-148.md +++ b/design/decisions/forks/OD-148.md @@ -52,7 +52,7 @@ No provisional reading is recorded, and no rebuild/restore operator entry was fo Admin access is the authority of an operation over the whole deployment. Every operator entry is an internal function, so only admin access reaches it. It takes the operator as a required argument, a string bounded as an actor's ID is, refuses an empty one, reads no grant, and writes that operator on what it changes: the generation row, the gate's audit record, the restore's run record. The Specs say "records the stated operator" and stop saying "authorizes an operator actor". No scope beyond the tenant is added. D11's one trusted deployment is the boundary, and the Spec says a stated operator is what admin access chose to state. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-150.md b/design/decisions/forks/OD-150.md index 0bd371e..42c1306 100644 --- a/design/decisions/forks/OD-150.md +++ b/design/decisions/forks/OD-150.md @@ -42,7 +42,7 @@ The row has no provisional reading; the lean keeps the Spec silent on a duration The write-pause Spec states no acceptable length. Before a paused rebuild starts, its length is estimated as the sum the lean names, backfill batches times batch time plus verify batches times batch time, and the number comes from the first paused rebuild that is measured on the native backend. Whether that length fires the doc's trigger for an online history rebuild is asked of the owner then, with the number. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-151.md b/design/decisions/forks/OD-151.md index cb1e28c..1b70972 100644 --- a/design/decisions/forks/OD-151.md +++ b/design/decisions/forks/OD-151.md @@ -43,7 +43,7 @@ The provisional reading keeps `allocated` as one total. [design/decisions/regist Stock state keeps totals only. `release` takes the quantities from the cancelled order's own lines, in the same mutation as the order's `cancel`, with Orders called first (OD-074). The stock decider rejects a release larger than the allocated total. A second cancel of one order is rejected by the order's own state, so no stock item needs to remember an order. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-152.md b/design/decisions/forks/OD-152.md index e5c833b..5fd6810 100644 --- a/design/decisions/forks/OD-152.md +++ b/design/decisions/forks/OD-152.md @@ -43,7 +43,7 @@ The provisional reading includes the history view under a write pause. [design/s The order allocation history view stays in the first experiment, because Sc L2-6 is a row of the doc and the experiment passes on every Layer 0 to 2 row. Its rules are written first (roadmap 4c) and its build is a unit of round 5, after the online rebuild. The roadmap's deferral "at a view that needs them" now covers the aggregate form only. -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/forks/OD-153.md b/design/decisions/forks/OD-153.md index db19c7a..e861cb0 100644 --- a/design/decisions/forks/OD-153.md +++ b/design/decisions/forks/OD-153.md @@ -43,7 +43,7 @@ The provisional reading is local native runs with production configuration. [des Sc L2-9 reads as the local observation: the production composition on the pinned local backend, with the same authority, schemas, concurrency and code path a release has. No Spec claims parity with a hosted deployment. The concurrency half is the contention run on the production composition, as OD-007's checked lean says. A hosted run stays deferred with Probe 3's quota half (OD-036). -Decided by the session on the owner's delegation, 2026-10-02. The main thread ruled it on the owner's words of 2026-10-02: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." The owner reopens it by naming it. +Ruled by the main thread on the owner's words of 2026-10-02 ("Decide on the forks in the roadmap."), and the owner agreed the same day to record the rulings as the owner's ("i agree with the least noisy and cleanest path forward."). ## Notes for the main thread diff --git a/design/decisions/register.json b/design/decisions/register.json index 6ca2bb8..ecacbdb 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -992,7 +992,7 @@ "provisionalReading": "Taken: the library registers no grant function; an operator runs an internal mutation with admin access.", "blocks": "S4", "advisor": "operator", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Owner tests 2 and 5: who holds the root of authority in a tenant is a promise to an adopter and trades safety against ease of adoption, and the corpus holds no evidence on which weighs more; doubt stated, because the mechanism beneath it, plain helpers and a composition's internal mutations, is delegated-shaped.", @@ -1002,9 +1002,9 @@ }, "checkedBy": null, "fork": "design/decisions/forks/OD-025.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "The first grant of a tenant is created with admin access through the composition's own internal grant mutation, and later grants the same way or by whatever command the composition registers. The library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation: those are an adopter's to build on the helpers. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-026", @@ -1833,7 +1833,7 @@ "provisionalReading": null, "blocks": "S3", "advisor": "product", - "status": "waiting", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, product direction: the largest order the platform promises an adopter and the targets it holds itself to, and OQ3 is the doc's own question (design/README.md:247); the maximum is also half of OD-123 (domain), which the number ruled here settles for both; the targets have no other row.", @@ -1843,9 +1843,9 @@ }, "checkedBy": null, "fork": "design/decisions/forks/OD-046.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "One answer for the largest order: 100 lines is the largest order `PlaceOrder` of the example supports, and the bound its declaration carries. It is the example's promise and no platform limit: a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings. One document per stream stays the default mapping, with the derived mapping for state that outgrows its budget. The Specs stop calling 100 provisional. The speed targets are set here, before the benchmark, as the doc requires. They are targets for the pinned local backend on the machine of the run; a hosted deployment is another question. When it set them the main thread had seen three single local runs, 51.6, 39.1 and 184.4 ms at 1, 10 and 100 lines, and the record says so. Latency without contention, the execution time of the top-level mutation as the backend's log reports it, over at least 30 accepted commands per size after 5 discarded: median at most 100 ms at 1 line, 100 ms at 10 lines and 400 ms at 100 lines, and 95th percentile at most 200, 200 and 800 ms. Throughput without contention: 8 callers at once, each placing ten-line orders on stock no other caller touches, complete at least 20 orders a second over at least 10 seconds, with the local backend's write cap of 4 MiB a second stated beside the number. Contention, N commands at once on one stock item with stock for one: at N of 2 and 8 every command reaches a business outcome, one applied and the rest rejected for short stock, and the engine fails none after its bounded retries; at N of 32 the same count is reported, and a command the engine failed is a recorded cost and no failure of the experiment, because the caller repeats it under the same request key. A target that is missed is reported as not held with its number. No target is changed after the run. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-047", @@ -1863,16 +1863,16 @@ "provisionalReading": null, "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, said in public: the paths are in a public repository's history, which an edit does not take back, and STATE.md:263 says the section is the owner's; nearer the owner when unsure, since the edit itself is layout.", "lean": null, "checkedBy": null, "fork": "design/decisions/forks/OD-047.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "The two references in `AGENTS.md` under \"On the owner's machine\" stay. They are paths and hold no content of the files. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-048", @@ -2528,7 +2528,7 @@ "provisionalReading": "OQ3: the largest order `PlaceOrder` supports is a product decision; the experiment measures with a provisional maximum of 100 lines and reports the numbers per line so the owner can choose (OQ3)", "blocks": "S3", "advisor": "convex", - "status": "waiting", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, product direction: the doc calls the largest order a product decision [design/specs/application/first-experiment.sdp.md:60,109].", @@ -2538,9 +2538,9 @@ }, "checkedBy": null, "fork": "design/decisions/forks/OD-064.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "One answer for the largest order: 100 lines is the largest order `PlaceOrder` of the example supports, and the bound its declaration carries. It is the example's promise and no platform limit: a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings. One document per stream stays the default mapping, with the derived mapping for state that outgrows its budget. The Specs stop calling 100 provisional. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-065", @@ -2706,7 +2706,7 @@ "provisionalReading": "Extension E-8: the doc names a generation registry, a marker and a gate without their shapes, and says to keep the old generation for a rollback period without saying whether live commands keep writing it; the option taken here is the `generations` and `projectionMarkers` tables below with seven states, a rollback period of 7 days, one building generation per read model at a time, and a retired generation that is kept but not written, so that a rollback reopens it as `verifying` and one verify pass repairs the rows changed since the switch before it is active again; the other reading is write-through, where live commands keep writing the retired generation for the whole period so that rollback is one write, at the cost of a third point read and write per read model per command for 7 days after every cutover; the cheaper steady path is taken and the owner rules whether the product needs instant rollback (E-8, D9, Sc L2-5)", "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, trades one value for another with no evidence which weighs more: a third point read and write per read model per command for 7 days against an instant rollback, and whether the product needs instant rollback is product direction [design/specs/application/generation-registry.sdp.md:33]. The tables, the seven states and the period are mechanism and go in the fork as the lean.", @@ -2715,9 +2715,9 @@ }, "checkedBy": null, "fork": "design/decisions/forks/OD-068.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "A retired generation is not written. Rollback reopens it as `verifying`, runs one verify pass that repairs what the intervening writes changed, and then switches. The switch is one write in both directions; the Specs stop promising a rollback of one write, and say rollback is a verify pass and then a switch of one write. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-069", @@ -2744,7 +2744,7 @@ "provisionalReading": "Extension E-8, reach of a generation: the doc says nothing about tenants and rebuild; the option taken here is one generation per read model spanning every tenant, so the registry has no tenant dimension, one row is active for all tenants and the batches iterate the tenant list; the other reading is one registry per read model and tenant, keyed by `readModel`, `tenantId` and `generation`, which lets one tenant rebuild while another keeps reading and lets a write pause close one tenant at a time, at the cost of one generation row, switch, rollback and purge per tenant, a projection-version rollout that is one generation per tenant, and a third index field on every registry read; the deployment-wide reading is taken because a generation is a build of the projection's code and the owner rules whether per-tenant rebuilds are a product need; a consequence the owner confirms with it is that the `generations` table is operational data whose scope is the whole deployment, not tenant data, so its two indexes `by_read_model` and `by_read_model_state` lead with the read model name rather than `tenantId`, a recorded deviation from Law 11's index rule and the review rubric's item 25, of the same kind the obligation module records under E-57, while the marker table, which is tenant data, leads with the tenant (E-8, D9, D11, Law 11, E-57)", "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, goes against a law of the doc and is product direction: the registry's two indexes lead with the read model and not the tenant, a recorded deviation from Law 11's index rule, and whether a per-tenant rebuild is a product need is the owner's [design/specs/application/generation-registry.sdp.md:34]. The fork gives the deployment-wide generation as the lean.", @@ -2753,9 +2753,9 @@ }, "checkedBy": null, "fork": "design/decisions/forks/OD-069.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "One generation per read model spans every tenant. The generation registry is operational data of the deployment and its indexes lead with the read model. Every table that holds tenant data, the read-model rows and the markers among them, still leads with the tenant. The deviation from Law 11's index rule is recorded where it is today, as confirmed. A rebuild for one tenant is not offered. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-070", @@ -5092,7 +5092,7 @@ "provisionalReading": "OQ3: one document per stream as the default, and the largest order the placement command supports, are product decisions; the design assumes one document and leaves the maximum lines as a placeholder in the first experiment (OQ3, D10)", "blocks": "S3", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, product direction and an open question of the doc: the largest order the placement command supports is OQ3 [design/specs/kernel/state-document-mapping.sdp.md:28], and S3 needs it before its benchmarks [design/STATE.md:139], a pointer; the 1, 10 and 100-line measurement already exists [design/STATE.md:158], a pointer.", @@ -5101,9 +5101,9 @@ }, "checkedBy": null, "fork": "design/decisions/forks/OD-123.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "One answer for the largest order: 100 lines is the largest order `PlaceOrder` of the example supports, and the bound its declaration carries. It is the example's promise and no platform limit: a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings. One document per stream stays the default mapping, with the derived mapping for state that outgrows its budget. The Specs stop calling 100 provisional. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-124", @@ -6089,7 +6089,7 @@ "provisionalReading": null, "blocks": "S4", "advisor": "operator", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Owner tests 5 and 6: who may run a rebuild or a restore across every tenant trades safety against ease with no evidence on which weighs more, and any grant for it needs a scope beyond the tenant, which rules on the term grant, a permission in a tenant [CONTEXT.md:128]; doubt stated, because admin access alone is a mechanism and D11's trust boundary already admits it.", @@ -6101,9 +6101,9 @@ }, "checkedBy": null, "fork": "design/decisions/forks/OD-148.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "Admin access is the authority of an operation over the whole deployment. Every operator entry is an internal function, so only admin access reaches it. It takes the operator as a required argument, a string bounded as an actor's ID is, refuses an empty one, reads no grant, and writes that operator on what it changes: the generation row, the gate's audit record, the restore's run record. The Specs say \"records the stated operator\" and stop saying \"authorizes an operator actor\". No scope beyond the tenant is added. D11's one trusted deployment is the boundary, and the Spec says a stated operator is what admin access chose to state. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-149", @@ -6159,7 +6159,7 @@ "provisionalReading": null, "blocks": "S4", "advisor": "operator", - "status": "waiting", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Owner test 5, and the doc makes acceptability itself a trigger: [docs/convex-transactional-domain-platform-decisions.md:354] activates online rebuild of cross-stream views when the write pause is unacceptable for them, so naming the acceptable length settles a trigger of the doc, which only the owner does; a policy by the operator lens's own rule.", @@ -6170,9 +6170,9 @@ }, "checkedBy": null, "fork": "design/decisions/forks/OD-150.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "The write-pause Spec states no acceptable length. Before a paused rebuild starts, its length is estimated as the sum the lean names, backfill batches times batch time plus verify batches times batch time, and the number comes from the first paused rebuild that is measured on the native backend. Whether that length fires the doc's trigger for an online history rebuild is asked of the owner then, with the number. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-151", @@ -6194,16 +6194,16 @@ "provisionalReading": "the stock item holds allocated as one total", "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Owner test 2, the example's policy under E-46, which the Spec leaves to the owner; raised by the domain advisor's roadmap memo of 2026-10-02 and made a row by the session under decision 36.", "lean": null, "checkedBy": null, "fork": "design/decisions/forks/OD-151.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "Stock state keeps totals only. `release` takes the quantities from the cancelled order's own lines, in the same mutation as the order's `cancel`, with Orders called first (OD-074). The stock decider rejects a release larger than the allocated total. A second cancel of one order is rejected by the order's own state, so no stock item needs to remember an order. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-152", @@ -6226,16 +6226,16 @@ "provisionalReading": "the history view is in the first experiment and its rebuild pauses writes", "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Owner test 2, the scope of the experiment; raised by the domain advisor's roadmap memo of 2026-10-02 and made a row by the session under decision 36.", "lean": null, "checkedBy": null, "fork": "design/decisions/forks/OD-152.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "The order allocation history view stays in the first experiment, because Sc L2-6 is a row of the doc and the experiment passes on every Layer 0 to 2 row. Its rules are written first (roadmap 4c) and its build is a unit of round 5, after the online rebuild. The roadmap's deferral \"at a view that needs them\" now covers the aggregate form only. Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." }, { "id": "OD-153", @@ -6260,16 +6260,16 @@ "provisionalReading": "Sc L2-9 is bound to native runs on a local backend with production configuration", "blocks": "S3", "advisor": "convex", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Owner test 1, the reading of a scenario of the doc; OD-007's lean carries the case against; raised by the convex advisor's roadmap memo of 2026-10-02 and made a row by the session under decision 36.", "lean": null, "checkedBy": null, "fork": "design/decisions/forks/OD-153.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-02", + "ruling": "Sc L2-9 reads as the local observation: the production composition on the pinned local backend, with the same authority, schemas, concurrency and code path a release has. No Spec claims parity with a hosted deployment. The concurrency half is the contention run on the production composition, as OD-007's checked lean says. A hosted run stays deferred with Probe 3's quota half (OD-036). Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." } ], "possiblySame": [ From 8e603b748d6f95aecccfc15a1f9629b0756ec195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:09:36 +0200 Subject: [PATCH 010/155] test: observe native scheduler limits and snapshot replacement --- design/README.md | 7 +- ...uled-functions.probe-7-fresh-import.sdp.md | 28 ++ ...d-functions.probe-7-in-place-import.sdp.md | 30 ++ ...exclude-pending-scheduled-functions.sdp.md | 15 +- ...tions-have-limits.probe-7-arguments.sdp.md | 29 ++ .../facts/f13-transactions-have-limits.sdp.md | 6 +- ...able-shows-failed-runs.probe-7-scan.sdp.md | 30 ++ ...le-shows-failed-runs.probe-7-states.sdp.md | 28 ++ ...d-functions-table-shows-failed-runs.sdp.md | 13 +- design/specs/facts/probe-plan.sdp.md | 8 +- design/specs/platform/native-harness.sdp.md | 4 +- harness/admin.ts | 14 +- harness/snapshot.ts | 12 +- ...led-functions.probe-7-fresh-import.test.ts | 33 ++ ...-functions.probe-7-in-place-import.test.ts | 33 ++ ...ions-have-limits.probe-7-arguments.test.ts | 29 ++ ...ble-shows-failed-runs.probe-7-scan.test.ts | 32 ++ ...e-shows-failed-runs.probe-7-states.test.ts | 32 ++ tests/native/scheduler-cases.ts | 347 ++++++++++++++++++ tests/native/scheduler-composition.ts | 132 +++++++ tests/native/scheduler-functions.ts | 159 ++++++++ tests/native/scheduler-restore.ts | 189 ++++++++++ tests/native/snapshot-archive.ts | 34 ++ tests/native/snapshot-replacement.test.ts | 18 +- 24 files changed, 1232 insertions(+), 30 deletions(-) create mode 100644 design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md create mode 100644 design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md create mode 100644 design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md create mode 100644 design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md create mode 100644 design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md create mode 100644 tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts create mode 100644 tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts create mode 100644 tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts create mode 100644 tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts create mode 100644 tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts create mode 100644 tests/native/scheduler-cases.ts create mode 100644 tests/native/scheduler-composition.ts create mode 100644 tests/native/scheduler-functions.ts create mode 100644 tests/native/scheduler-restore.ts create mode 100644 tests/native/snapshot-archive.ts diff --git a/design/README.md b/design/README.md index 96361e1..f23e1a6 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -202 specs · 5 packs · 101 anchors → 308 nodes · 1097 edges (0 errors, 0 warnings) +207 specs · 5 packs · 106 anchors → 318 nodes · 1112 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -316,6 +316,11 @@ The Layer 2 design added two more for the same reason, and nine examples that ve | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split` | Probe 5: a full first page under a row cap of twice the page size is not split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap` | Probe 5: the same page under a row cap equal to the page size is split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap` | Probe 5: a parent list's page outgrows its row cap | +| none | `spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import` | Probe 7: replacement into a fresh backend restores context data without schedules | +| none | `spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import` | Probe 7: replacement in place preserves schedules that run against restored data | +| none | `spec:facts.f13-transactions-have-limits.probe-7-arguments` | Probe 7: scheduled argument limits count encoded values | +| none | `spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-scan` | Probe 7: a failed-function scan reads unrelated schedules | +| none | `spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-states` | Probe 7: the five scheduler states remain readable | Package A therefore has 79 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md new file mode 100644 index 0000000..a146f21 --- /dev/null +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md @@ -0,0 +1,28 @@ +--- +id: spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f12-backups-exclude-pending-scheduled-functions + verifies: spec:facts.f12-backups-exclude-pending-scheduled-functions +--- +# Probe 7: replacement into a fresh backend restores context data without schedules + +Probe 7 · native backend tier · temporary copy of the production composition. + +## Intent + +- outcome: Replacement into a fresh backend restores context data without schedules. (Probe 7) + +```gwt +Given the production composition with temporary scheduler functions +When the native backend exercises fresh-import +Then the observation is {result: "documents preserved and schedules absent"} +``` + +## Verification — executable + +- The archive carries parent, Orders and Inventory documents with their IDs and creation times, but no scheduler rows. Destination environment variables remain unchanged. Stored scheduler IDs validate at import, on a later write and as an argument; system.get returns null and cancel succeeds without a row. These are expectations before the first run. Hosted dashboard restore, Workpool and Workflow are not run. +- Bound values are expectations written before the first native run; measured durations and sizes are recorded, not asserted. +- The first native run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` held the original bound values. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md new file mode 100644 index 0000000..b9008b7 --- /dev/null +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md @@ -0,0 +1,30 @@ +--- +id: spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f12-backups-exclude-pending-scheduled-functions + verifies: spec:facts.f12-backups-exclude-pending-scheduled-functions +--- +# Probe 7: replacement in place preserves schedules that run against restored data + +Probe 7 · native backend tier · temporary copy of the production composition. + +## Intent + +- outcome: Replacement in place preserves schedules that run against restored data. (Probe 7) + +```gwt +Given the production composition with temporary scheduler functions +When the native backend exercises in-place-import +Then the observation is {result: "documents replaced and schedules preserved"} +``` + +## Verification — executable + +- Every scheduler state in the parent and both contexts survives replacement. A pending reaction created after export still runs at its scheduled time against restored data. A failed reaction created after export remains visible despite having no matching exported data. The system table alone cannot identify which intent belongs to the exported data. Hosted dashboard restore, Workpool and Workflow are not run. +- Bound values are expectations written before the first native run; measured durations and sizes are recorded, not asserted. +- A kept pending row referenced by an imported `v.id("_scheduled_functions")` is canceled after import; the cancellation is expected to prevent that reaction, while the uncanceled row created after export still runs against restored data. +- The first native run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` held the original bound values. +- The uncanceled reaction is expected to commit no earlier than its recorded scheduled time; elapsed waiting time is recorded rather than bounded. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md index 70cef68..19908e1 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md @@ -10,15 +10,26 @@ relations: F12 · Status: documented · Doc status: Documented, S8 · Decisions: D13, D19. -A backup holds the tables and file storage; it excludes code and configuration, pending scheduled functions and environment variables, and a restore wipes existing data and replaces it with the backup. A restored deployment therefore has obligations but no dispatches, which is one of the four reasons the obligation table earns its cost and why restore starts with dispatch off and rebuilds schedules from obligations. The page does not say what restore does to component data or to the scheduler, so Probe 7 keeps that question. +The backup page excludes scheduled functions, code, configuration and environment variables. On the pinned native backend, CLI snapshot export and replacement import carry the parent, Orders and Inventory documents, including their IDs and creation times, but no scheduler table. Import into a fresh backend leaves its scheduler tables empty; import in place preserves its scheduler rows, including rows created after export. The inference that every restore leaves no dispatches does not hold for this CLI import. The destination environment variables are unchanged. Hosted dashboard restore is not run, so the fact retains documented status for that wider claim. Here snapshot means the exported archive, not current state or a baseline event. ## Intent - outcome: Record that backups exclude pending scheduled functions, with its evidence status as a target, so that every design resting on it is found by its `constrainedBy` edge (F12) +### Open questions + +- [non-blocking] Hosted dashboard restore, file storage, Workpool and Workflow are not run; local CLI import does not settle those paths. (F12, Probe 7) + ## Constraints - statement: Backups exclude pending scheduled functions (F12) - flavor: convex-fact - target: evidence.status:documented -- measurableBy: S8 https://docs.convex.dev/database/backup-restore, read again 2026-09-30; doc status Documented, S8; Probe 7 shows what a restore leaves of Workpool, Workflow and scheduler state (F12, D13, D19) +- measurableBy: S8 https://docs.convex.dev/database/backup-restore and https://docs.convex.dev/database/import-export/import, read 2026-10-02; native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0; `npx vitest run --project native tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts --testTimeout 300000` as part of `evidence/runs/native-20261002T205553Z-592f1dc-97c74164-c3a8-4818-9f77-2f04b1456261.json`, both import cases passed; the same record includes unrelated probe failures (F12, D13, D19) + +## Design + +- nativeRestore: replacement into both destinations restores the observed parent, Orders and Inventory tables exactly; archive entries contain no `_scheduled_functions`; all five scheduler states in each scope remain in place, and none appear in the fresh destination (F12, Probe 7) +- schedulerReferences: a stored `v.id("_scheduled_functions")` imports into the fresh destination, validates as a function argument and on another document write, returns null from `ctx.db.system.get`, and `ctx.scheduler.cancel` succeeds without making a row; in place the same ID resolves to the kept row and cancellation prevents that pending reaction; an uncanceled row still executes against restored data (F12, Probe 7) +- keptReactions: pending mutations created after export survive import in place and execute at their scheduled time against the restored data; a failed row created after export also survives while its document reference does not, so the system table alone does not say which intent belongs to the exported data (F12, D13, D19, Probe 7) +- statusScope: local CLI export and replacement are observed at the native backend tier; documented status remains because hosted dashboard backup and restore are not run, and the local observation is not a confirmed defect in that path (F12, Probe 7) diff --git a/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md b/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md new file mode 100644 index 0000000..15db15d --- /dev/null +++ b/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md @@ -0,0 +1,29 @@ +--- +id: spec:facts.f13-transactions-have-limits.probe-7-arguments +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f13-transactions-have-limits + verifies: spec:facts.f13-transactions-have-limits +--- +# Probe 7: scheduled argument limits count encoded values + +Probe 7 · native backend tier · temporary copy of the production composition. + +## Intent + +- outcome: Scheduled argument limits count encoded values. (Probe 7) + +```gwt +Given the production composition with temporary scheduler functions +When the native backend exercises arguments +Then the observation is {result: "16 MiB summed Convex value bytes per mutation; 4 MiB per call warns"} +``` + +## Verification — executable + +- The original expectation before the first run followed the limits page. The test constructs values inside the handler, tries ASCII and multibyte strings and arrays of small strings, records refusal text and whether runAfter returned, and fails unless it reaches a refusal and a successful boundary neighbor. Hosted limits are not run. +- Original bound values were written before the first native run; the bound values here follow the observed refusal. Measured durations are recorded, not asserted. +- The first run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` accepted calls above 4 MiB with a warning and refused the sum above 16 MiB at `runAfter`; the original 4 MiB hard-limit expectation did not hold. These values are observed. Exact-byte and encoding comparisons extend that observation. +- At the enforced boundary the test binds 16777216 accounted bytes, accepting one ASCII string of 16777202 bytes, 8388601 copies of é, 1024 strings totaling 16775154 content bytes, and eight calls each carrying 2097138 content bytes; their first larger tested neighbors refuse. The tested object costs 14 accounted bytes beside its string, or 14 plus two per array element beside its strings. Newline and quote characters distinguish this from JSON encoding. diff --git a/design/specs/facts/f13-transactions-have-limits.sdp.md b/design/specs/facts/f13-transactions-have-limits.sdp.md index a725fb2..7572fea 100644 --- a/design/specs/facts/f13-transactions-have-limits.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.sdp.md @@ -25,7 +25,7 @@ The numbers below were read from the limits page on 2026-09-30 and match the pla ## Design -The numbers are the ceilings the design treats as such; every batch states a bound below them. The page's values were read on 2026-09-30 and match the plan's section 9. +The documented numbers bound batches. Probe 4 observes transaction budgets; Probe 7 distinguishes the scheduled-argument page values from the pinned backend's enforced budget. A warning on this release does not remove the documented ceiling from the design. - limitDocument: 1 MiB per document, 1024 fields, nesting depth 16, 8192 array elements (F13, S9) - limitTransactionRead: 16 MiB read, 32,000 documents scanned and 4,096 index ranges read per transaction (F13, S9) @@ -33,8 +33,10 @@ The numbers are the ceilings the design treats as such; every batch states a bou - limitFunctionPayload: 16 MiB per function argument set and 16 MiB per return value; a Node action's arguments are capped at 5 MiB (F13, S9) - limitQueryMutationTimeout: 1 second of execution per query or mutation (F13, S9) - limitActionTimeout: 30 minutes in the Convex runtime and 10 minutes in the Node runtime (F13, S9) -- limitScheduling: 1000 scheduled functions per mutation, 4 MiB per scheduled argument set, 16 MiB of scheduled arguments per mutation, 1,000,000 outstanding scheduled functions (F13, S9) +- limitScheduling: the limits page states 1000 scheduled functions per mutation, 4 MiB per scheduled argument set, 16 MiB summed arguments per mutation and 1000000 outstanding functions; the scheduling page states 8 MB total, both read 2026-10-02; the pinned native backend warns above 4 MiB per call, accepts calls above both 8 MB and 8 MiB, and refuses the sum above 16 MiB at `runAfter` (F13, S9, Probe 7) - limitSchema: 32 indexes per table, 16 fields per index, 10,000 tables per deployment (F13, S9) - limitNestedCalls: the limits page states no count of nested `runQuery` or `runMutation` calls per function; the page on writing data states that nested calls share the overall transaction limits, and Probe 4 showed the same across a component boundary for bytes read and bytes written; the pinned backend commits a call stack of nine functions and fails the tenth with "Cross component call depth limit exceeded", which is a backend default and not a documented limit (F13, S9, Probe 4) - transactionMetrics: `ctx.meta.getTransactionMetrics()` is asynchronous and returns `used` and `remaining` per metric, and a nested call accepts `transactionLimits`; Probe 4 read the metrics in a parent around a child's read, nested and in a component, and they grew by what the child read (F13, Probe 4) - limitsAreCeilings: platform limits are ceilings, not batch sizes; every bulk operation, sweeper and batch states a bound tested against the pinned Convex version (D19, F13) +- schedulingObservation: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, `npx vitest run --project native tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts --testTimeout 300000` as part of `evidence/runs/native-20261002T205553Z-592f1dc-97c74164-c3a8-4818-9f77-2f04b1456261.json`; the original 4 MiB hard-limit expectation failed; values are constructed inside the mutation, and the caught `runAfter` refusal returns normally with the number of earlier successful schedules, distinguishing this refusal from commit (F13, Probe 7) +- schedulingBytes: native backend `precompiled-2026-09-28-5c7cb5b`, `npx vitest run --project native tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts` as part of `evidence/runs/native-20261002T205857Z-592f1dc-194ec93d-d96c-4eb7-b639-f7e2c630e648.json`, passed exact-byte boundaries: one string, multibyte text, 1024 strings and eight calls each fit at 16777216 accounted bytes and refuse their first larger tested value; for the tested `{ payload: string }` shape accounting is UTF-8 content bytes plus 14 per call, and for `{ payload: string[] }` it is content bytes plus 14 plus two per element per call; three newline bytes count as three, unlike six JSON escape bytes, so this is Convex value accounting, not JSON text length (F13, Probe 7) diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md new file mode 100644 index 0000000..ca9219f --- /dev/null +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md @@ -0,0 +1,30 @@ +--- +id: spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-scan +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f16-scheduled-functions-table-shows-failed-runs + verifies: spec:facts.f16-scheduled-functions-table-shows-failed-runs +--- +# Probe 7: a failed-function scan reads unrelated schedules + +Probe 7 · native backend tier · temporary copy of the production composition. + +## Intent + +- outcome: A failed-function scan reads unrelated schedules. (Probe 7) + +```gwt +Given the production composition with temporary scheduler functions +When the native backend exercises scan +Then the observation is {result: "4091 rows scan and 4092 exceed 4096 reads; zero document reads counted"} +``` + +## Verification — executable + +- A plain scheduled mutation records a local reaction. The failed-function filter reads unrelated outstanding rows. Metrics and completion usage record the scan cost. Hosted scan limits and seven-day expiry are not run. +- Original bound values were written before the first native run; the bound values here follow the observed refusal. Measured durations are recorded, not asserted. +- The first run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` counted zero document reads at 1000 rows; the next run refused 32000 rows for too many system operations. The original document-read expectation did not hold. The 4091 and 4092 boundary is an expectation derived from the recorded system-operation count before its first run. +- The boundary run on 2026-10-02 observed 4091 rows returning one failed reaction, with `databaseQueries.used` at 4096, and 4092 rows refused by the 4096-read limit; the same refusal also occurred at 32000, while a system-operation timeout occurred at 32001. The bound values are observed, and a plain scan without metric calls is also checked. +- In `evidence/runs/native-20261002T210145Z-592f1dc-1f503d30-03eb-431b-b48b-e946c2866bd5.json`, the plain scan without metric calls also accepted 4091 rows and refused 4092 for the 4096-read limit. Total rows include three terminal rows; outstanding rows are total rows minus three. diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md new file mode 100644 index 0000000..0cfd365 --- /dev/null +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md @@ -0,0 +1,28 @@ +--- +id: spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-states +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f16-scheduled-functions-table-shows-failed-runs + verifies: spec:facts.f16-scheduled-functions-table-shows-failed-runs +--- +# Probe 7: the five scheduler states remain readable + +Probe 7 · native backend tier · temporary copy of the production composition. + +## Intent + +- outcome: The five scheduler states remain readable. (Probe 7) + +```gwt +Given the production composition with temporary scheduler functions +When the native backend exercises states +Then the observation is {result: "all five states readable"} +``` + +## Verification — executable + +- Native observations include names, arguments, times and failure errors in the parent and both contexts. Completion rows remain readable during the measured interval. Seven-day expiry and hosted retention are not run. +- Bound values are expectations written before the first native run; measured durations and sizes are recorded, not asserted. +- The first native run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` held the original bound values. diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md index 628f363..41adbe6 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md @@ -10,7 +10,7 @@ relations: F16 · Status: rechecked · Doc status: Assumed in the doc; found documented on S6 by the recheck of 2026-09-30 · Decisions: D13. -The doc lists this fact as assumed. The lead read S6 on 2026-09-30 and this package read it again the same day: the `_scheduled_functions` system table carries `name`, `args`, `scheduledTime`, `completedTime` and `state`; the states are Pending, InProgress, Success, Failed and Canceled; and scheduled function results are available for 7 days after they have completed. The corpus records the recheck here and the target states it; the doc keeps its own status until the owner edits it, which is why `measurableBy` still names the doc's status. The do-nothing check of D13 depends on this fact: a plain scheduled mutation plus a scan of this table may cover a local reaction, and the obligation table must earn its cost through restore, retention past the 7-day window, and business visibility and operator exits. +The scheduled-functions page documents five states and a seven-day completion-result window. The pinned native backend exposes `pending`, `inProgress`, `success`, `failed` and `canceled` through admin access in the parent, Orders and Inventory. Rows carry `name`, `args`, `scheduledTime` and `state`; terminal rows carry `completedTime`; failed rows carry `state.error`. The native test observes a short retention interval only. Seven-day retention and expiry remain documented and are not run, so the fact retains rechecked status. The do-nothing check must also distinguish a fresh replacement, which has no scheduler intent, from replacement in place, which preserves intent created after export. ## Intent @@ -18,11 +18,18 @@ The doc lists this fact as assumed. The lead read S6 on 2026-09-30 and this pack ### Open questions -- [non-blocking] Probe 7 pending: the states and the 7-day retention are documented, so the probe need only confirm them on a native backend and show what a restore leaves of scheduler, Workpool and Workflow state (F16, Probe 7, F12) +- [non-blocking] Seven-day retention and expiry, hosted dashboard restore, Workpool and Workflow are not run; the short native retention observation does not establish the documented window. (F16, Probe 7, F12) ## Constraints - statement: `_scheduled_functions` shows failed runs for some retention window (F16) - flavor: convex-fact - target: evidence.status:rechecked -- measurableBy: S6 https://docs.convex.dev/scheduling/scheduled-functions, states Pending, InProgress, Success, Failed and Canceled, results available for 7 days after completion; doc status Assumed in the doc; found documented on S6 by the recheck of 2026-09-30; Probe 7 confirms the states and retention on a native backend and what a restore leaves (F16, D13) +- measurableBy: S6 https://docs.convex.dev/scheduling/scheduled-functions, read 2026-10-02; native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0; `npx vitest run --project native tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts`, `evidence/runs/native-20261002T205513Z-592f1dc-c8bbed5d-4726-482f-a875-5cde26210902.json`: states passed and scan expectation failed; doc status Assumed in the doc, states and seven-day retention documented on S6 (F16, D13) + +## Design + +- nativeStates: all five lowercase states are observed in each scope; a held action supplies `inProgress`, which is not a mutation state; completed rows remain readable across the recorded short observation interval (F16, Probe 7) +- scanFinding: the first native scan expected filtered-out scheduler rows to consume the 32000-document allowance; at 1000 rows it instead counted zero document reads and zero bytes, while `databaseQueries.used` grew by 1001; a subsequent run at 32000 rows failed with "Your request timed out performing too many system operations." in `evidence/runs/native-20261002T205553Z-592f1dc-97c74164-c3a8-4818-9f77-2f04b1456261.json`; this is an observed system-operation refusal, not evidence that 32000 is the scan boundary (F16, F13, Probe 7) +- scanBoundary: native backend `precompiled-2026-09-28-5c7cb5b`, `npx vitest run --project native tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts` as part of `evidence/runs/native-20261002T205857Z-592f1dc-194ec93d-d96c-4eb7-b639-f7e2c630e648.json`, observed 4091 rows scanning successfully and 4092 refused with "Too many reads in a single function execution (limit: 4096)"; bytes and documents read remain zero, but the scan adds N + 1 database queries to four already used; 32000 and 32001 are populated and refused, so the document ceiling is not the first boundary; the run failed because the expectation named only the system-operation timeout, and the example is rebound to the observed read refusal (F16, F13, Probe 7) +- scanConfirmation: `npx vitest run --project native tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts`, native backend `precompiled-2026-09-28-5c7cb5b`, `evidence/runs/native-20261002T210145Z-592f1dc-1f503d30-03eb-431b-b48b-e946c2866bd5.json`, passed both the instrumented and plain scan at 4091 total rows and refused both at 4092; three rows are terminal, so these correspond to 4088 and 4089 outstanding rows; the admin count also confirms 32001 total rows (F16, F13, Probe 7) diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index b83e22d..6cde207 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -20,7 +20,7 @@ relations: Feature · Detail: full transcription · Traces: Probe 1 to Probe 7, F4, F5, F12, F13, F14, F15, F16, F17, S13, E-16. -Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`, so Probe 7 need only confirm them and answer what a restore leaves; the backup page says nothing about component data or the scheduler, so that half of Probe 7 stands. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, and each result is recorded on its fact; probes 6 and 7 have not run. +Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`; Probe 7 observes the states and a short retention interval, scheduled-argument accounting, failed-function scan cost and local CLI replacement, while seven-day expiry and hosted dashboard restore remain open. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, and each result is recorded on its fact; Probe 7 has native observations for the local scheduler and CLI import, with the remaining scope named below; Probe 6 is separate. The probes are not the first experiment. The experiment builds Layers 0 to 2 and measures cost; the probes settle single facts and can run before it, in a fixture app that is separate from the example app. @@ -39,7 +39,7 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - [non-blocking] Probe 3 ran on 2026-10-01 for latency on a local backend; a local backend has no function-call quota, so that half waits for a hosted deployment (Probe 3, F4, D2, D8) - [non-blocking] Probe 5 ran on 2026-10-01: a parent query over a component query stays reactive, pages pinned by their end cursors stay contiguous, and a capped page crosses the boundary as `SplitRequired`; the `convex-helpers/react` hook then lost rows, and slice S2 decides how a context list is capped before it builds one (Probe 5, F15, S4, D8) - [non-blocking] Probe 6 pending: it must show a backfill batch racing a live command under optimistic concurrency, and that the migrations component fits a generation backfill (Probe 6, F17, D9) -- [non-blocking] Probe 7 pending: before Layer 3, it must show what a restore leaves of Workpool and Workflow state and confirm the states and retention of `_scheduled_functions` on a native backend (Probe 7, F12, F16, D13, D19) +- [non-blocking] Probe 7 observes all five scheduler states, local CLI replacement into a fresh backend and in place, stored scheduler IDs, kept reactions, scheduled-argument limits and failed-function scan cost; seven-day expiry, hosted dashboard restore, Workpool and Workflow remain open before the decisions resting on those observations. (Probe 7, F12, F13, F16, D13, D19) - [non-blocking] Extension E-16: the doc says each probe is one small test on a native backend and that a probed fact is one a native backend test showed, and does not say how a test that settles an unknown states what it expects or what follows when the backend answers otherwise; the option taken here: a probe is written as example Specs that verify its fact, each bound to its native test; an example's bound values are the expectation written before the first run; a first run that shows another value is a finding recorded on the fact, and the example is then bound to what that release does and says so; an example fails when its run did not reach the boundary it is about; a later run on a new release that shows another value is again a finding, and the test is not repaired to keep the old value; a size is recorded and not asserted, while a limit the backend enforces is bound as observed; and what a local backend cannot answer is named on the example and stays open on the fact; the owner confirms (E-16, Probes, Acceptance scenarios) ## Workflow @@ -61,7 +61,7 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - Probe 4 reads and writes near the per-transaction limits from a parent, from a nested mutation and from a component, and records whether the limits add up across the boundaries or apply per call; it serves D10 (Probe 4, F13) - Probe 5 subscribes to a parent query that calls a component query, changes the component's data, and checks that the subscription updates; it then pages a component list built with `paginator` through the parent while live writes land inside the paged range, with the client passing the end cursor as `convex-helpers` describes, records whether the pages stay contiguous, and fills one page's range past the component's `maximumRowsRead` so that a re-run comes back `SplitRequired`, recording whether the `convex-helpers/react` hook splits it across the boundary without a gap; it serves D8 (Probe 5, F15, S4) - Probe 6 runs a backfill batch through the migrations component while a live command writes the same read-model row, and checks that neither overwrites newer data and that the batch resumes after interruption; it serves D9 (Probe 6, F1, F17) -- Probe 7, before Layer 3, takes a backup with pending scheduled functions, Workpool jobs and a Workflow run in flight, restores it, and records what survives; it reads `_scheduled_functions` for a failed run and confirms the states and the retention window; it serves D13 and D19 (Probe 7, F12, F16) +- Probe 7 exports the parent, Orders and Inventory with all five scheduler states, replaces data into a fresh backend and in place, tests stored scheduler IDs and execution of kept pending mutations, measures scheduled-argument accounting and filtered scan cost, and records the short retention interval it actually observes; hosted dashboard restore, seven-day expiry, Workpool and Workflow are separate cases that remain open; it serves D13 and D19 (Probe 7, F12, F13, F16) ## Design @@ -70,7 +70,7 @@ Each probe is a fixture-app test on a disposable backend. The probe app is separ - backend: one disposable native backend per probe run, from the same local backend S13 describes (Acceptance scenarios, S13) - fixtureApp: the probe app is the kernel's fixture app, separate from the example app, and test-only functions never ship (Acceptance scenarios) - evidence: each run records commit, backend and dependency versions, configuration, command and result, and states any difference from production configuration (Acceptance scenarios) -- probe7Narrowed: the states and the 7-day retention of `_scheduled_functions` were found documented on 2026-09-30, so the probe confirms them and spends its effort on what a restore leaves (F16, S6, S8) +- probe7Narrowed: native scheduler and CLI import cases deploy temporary copies of the production composition from modules under `tests/native/`, with added functions and schema hashes recorded; no probe function ships in the example or fixture; local observations do not establish hosted restore or seven-day expiry (F12, F13, F16, S6, S8) - location: the probe app lives in this repository, beside the design, as the owner ruled under OQ4 on 2026-10-01 (OQ4) ## Verification — reviewed diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 551f56a..df37bc6 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -69,7 +69,7 @@ The harness is plain test support with no Convex function in it. The entries who - adminReads: [extension] the harness reads a table of the app or of a component, and the function log's completion records, with the admin key; a table read returns every document with every Convex value intact or fails, never a sample, and is several queries, not one snapshot; a log read returns every completion record after a point the test marks or fails, and a mark taken before a restart cannot be read after it; a completion record carries at least `identifier`, `componentPath`, `requestId`, `executionTime` in seconds, `cachedResult` and `error` (E-15) - typeFixtureIssuer: [extension] `interface FixtureIssuer { issuer: string; applicationID: string; jwks: string; token(subject: string): Promise }` (E-13) - typeBackend: [extension] `interface Backend { url: string; adminKey: string; issuer: FixtureIssuer; admin: AdminAccess; stop(): Promise; kill(): Promise; restart(): Promise; dispose(): Promise; facts(): BackendFacts }`; `stop` and `kill` keep the storage; `dispose` alone removes it and is final, so a disposed backend cannot be started again; `adminKey` is there for the one example that shows why no caller carries it (E-15) -- typeAdminAccess: [extension] `interface AdminAccess { deploy(): Promise; codegen(): Promise; setEnvironment(variables: Record): Promise; environment(): Promise>; run(name: string, args?: Record, options?: { component?: string }): Promise; readTable(table: string, options?: { component?: string; pageSize?: number }): Promise[]>; writeTable(table: string, write: { insert: Record } | { patch: string; fields: Record } | { delete: string }, options?: { component?: string }): Promise; logMark(): Promise; completionsSince(mark: LogMark, until: (records: readonly CompletionRecord[]) => boolean, timeoutMs?: number): Promise; exportSnapshot(path: string): Promise; replaceSnapshot(path: string): Promise }` holds every use of the admin key and takes no identity; the two snapshot members run the Convex CLI's export and import with replacement against this backend, and a test uses them and never the admin key itself (E-15) +- typeAdminAccess: [extension] `interface AdminAccess { deploy(): Promise; deployTemporary(directory: string): Promise<{ command: string[]; stdout: string; stderr: string }>; codegen(): Promise; setEnvironment(variables: Record): Promise; environment(): Promise>; run(name: string, args?: Record, options?: { component?: string }): Promise; readTable(table: string, options?: { component?: string; pageSize?: number }): Promise[]>; writeTable(table: string, write: { insert: Record } | { patch: string; fields: Record } | { delete: string }, options?: { component?: string }): Promise; logMark(): Promise; completionsSince(mark: LogMark, until: (records: readonly CompletionRecord[]) => boolean, timeoutMs?: number): Promise; exportSnapshot(path: string): Promise; replaceSnapshot(path: string): Promise }` holds every use of the admin key and takes no identity; the two snapshot members run the Convex CLI's export and import with replacement against this backend, and a test uses them and never the admin key itself; `deployTemporary` deploys a temporary copy, returns its redacted command and clears the plain composition claim (E-15) - fnOrdinaryClient: [extension] `ordinaryClient(url: string, options?: { token?: string; fetch?: typeof fetch }): ConvexHttpClient` (E-13) - fnOrdinarySocketClient: [extension] `ordinarySocketClient(url: string, options?: { token?: string; webSocket?: typeof WebSocket }): ConvexClient` (E-13) - fnCountingFetch: [extension] `countingFetch(options: { loseResponses: boolean }): { fetch: typeof fetch; requests(): number }` delivers each request and, when it loses responses, throws after the backend has answered (E-15) @@ -78,7 +78,7 @@ The harness is plain test support with no Convex function in it. The entries who - fnFixtureBackend: [extension] `fixtureBackend(options?: { issuer?: string; executable?: Executable; parentDirectory?: string; deploy?: boolean }): Promise` starts a backend, deploys the fixture composition unless `deploy` is `false`, and removes the backend when the test finishes, however it finishes; the executable and parent directory options let the harness's own tests exercise the same cleanup path (E-15, E-13) - fnProductionBackend: [extension] `productionBackend(options?: { issuer?: string; deploy?: boolean }): Promise` starts a backend, deploys the production composition unless `deploy` is `false`, and removes the backend when the test finishes, however it finishes (E-15, E-13) - fnMeasure: [extension] `measure(name: string, value: JsonValue): void` stores a measurement on the running test (E-15) -- typeBackendFacts: [extension] `interface BackendFacts { composition: "fixture" | "production" | null; installedLayers: string[]; executable: { release: string | null; sha256: string; source: "cache" | "CONVEX_BACKEND_BINARY" }; identitySource: { kind: "fixture issuer"; issuer: string }; environment: string[]; dataset: "empty at start" }` (E-15, E-13) +- typeBackendFacts: [extension] `interface BackendFacts { composition: "fixture" | "production" | null; installedLayers: string[]; executable: { release: string | null; sha256: string; source: "cache" | "CONVEX_BACKEND_BINARY" }; identitySource: { kind: "fixture issuer"; issuer: string }; environment: string[]; dataset: "empty at start" }` uses null after a temporary deployment; its run measurement names the base composition, added functions and schema hashes instead of claiming the plain composition was deployed (E-15, E-13) - typeNativeTestEntry: [extension] `interface NativeTestEntry { project: "types" | "pure" | "simulator" | "native"; name: string; file: string; result: "passed" | "failed" | "skipped"; durationMs: number | null; errors: string[]; backends: BackendFacts[]; measurements: { name: string; value: JsonValue }[] }`; a run that includes the native project records every test it ran, of every project, each under the project that ran it (E-15) - typeNativeRunRecord: [extension] `interface NativeRunRecord { tier: "native"; commit: string; clean: boolean; command: string; startedAt: string; finishedAt: string; result: "passed" | "failed" | "interrupted"; versions: Record; tests: NativeTestEntry[]; unhandledErrors: string[] }`; it holds no field for adjusted configuration, and a run that adjusts configuration is the trigger that adds one (E-15, Acceptance scenarios) - compositionLayout: [extension] a composition is a name, a project directory and a functions directory; the Convex CLI reads `convex.json` and `package.json` from its working directory and takes no functions directory any other way, so the harness runs every CLI call in the composition's project directory; the fixture composition's project directory is the repository root, whose `convex.json` names `fixture/convex/`, and the production composition's is `example/`, whose `convex.json` names `convex/` and whose `package.json` is a symbolic link to the root's, so the repository has one `package.json` (E-15) diff --git a/harness/admin.ts b/harness/admin.ts index c7c6e5d..5ae1ac3 100644 --- a/harness/admin.ts +++ b/harness/admin.ts @@ -10,7 +10,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { redact, runChild } from "./child.js"; -import { createSnapshotAccess } from "./snapshot.js"; +import { snapshotCommand, createSnapshotAccess } from "./snapshot.js"; +import type { SnapshotCommand } from "./snapshot.js"; import { projectDirectory } from "./composition.js"; import type { Composition } from "./composition.js"; const anchor = codeAnchor({ @@ -54,6 +55,7 @@ export interface CompletionRecord { } export interface AdminAccess { deploy(): Promise; + deployTemporary(directory: string): Promise; codegen(): Promise; setEnvironment(variables: Readonly>): Promise; environment(): Promise>; @@ -212,6 +214,16 @@ export function createAdminAccess( await cli("deploy"); state.deployed = target.composition; }, + async deployTemporary(directory) { + const result = await snapshotCommand( + target, + [...commandArgs.deploy], + target.signal, + directory, + ); + state.deployed = undefined; + return result; + }, codegen: () => cli("codegen"), async setEnvironment(variables) { const response = await fetch( diff --git a/harness/snapshot.ts b/harness/snapshot.ts index cc33ab0..e01de14 100644 --- a/harness/snapshot.ts +++ b/harness/snapshot.ts @@ -23,7 +23,7 @@ export interface SnapshotCommand { } // Uses the installed CLI and an isolated home, with no inherited deployment selection. -async function snapshotCommand( +export async function snapshotCommand( backend: Pick, args: string[], signal: AbortSignal | undefined, @@ -54,16 +54,12 @@ async function snapshotCommand( } export function deploySnapshotFixture( - backend: Pick, + backend: Pick, directory: string, signal: AbortSignal, ): Promise { - return snapshotCommand( - backend, - ["deploy", "--yes", "--codegen", "disable", "--typecheck", "disable"], - signal, - directory, - ); + signal.throwIfAborted(); + return backend.admin.deployTemporary(directory); } export function createSnapshotAccess( diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts new file mode 100644 index 0000000..eda3057 --- /dev/null +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts @@ -0,0 +1,33 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe7FreshImportContract as contract } from "../../generated/contracts/facts.f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.contract.js"; +import { schedulingBackend } from "./scheduler-composition.js"; +import type { SchedulerWorld } from "./scheduler-cases.js"; +import { freshImport } from "./scheduler-restore.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import", + ), + verifies: ref( + "spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import", + ), +}); +void anchor; +bindExample(contract, (): SchedulerWorld => ({}), { + "the production composition with temporary scheduler functions": async ( + world, + ) => { + Object.assign(world, await schedulingBackend()); + }, + "the native backend exercises fresh-import": async (world) => { + world.result = await freshImport(world.backend!, world.directory!); + }, + "the observation is {result}": (world, { result }) => { + expect(world.result).toBe(result); + }, +}); diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts new file mode 100644 index 0000000..bd64caa --- /dev/null +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts @@ -0,0 +1,33 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe7InPlaceImportContract as contract } from "../../generated/contracts/facts.f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.contract.js"; +import { schedulingBackend } from "./scheduler-composition.js"; +import type { SchedulerWorld } from "./scheduler-cases.js"; +import { inPlaceImport } from "./scheduler-restore.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import", + ), + verifies: ref( + "spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import", + ), +}); +void anchor; +bindExample(contract, (): SchedulerWorld => ({}), { + "the production composition with temporary scheduler functions": async ( + world, + ) => { + Object.assign(world, await schedulingBackend()); + }, + "the native backend exercises in-place-import": async (world) => { + world.result = await inPlaceImport(world.backend!, world.directory!); + }, + "the observation is {result}": (world, { result }) => { + expect(world.result).toBe(result); + }, +}); diff --git a/tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts b/tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts new file mode 100644 index 0000000..366f5b7 --- /dev/null +++ b/tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts @@ -0,0 +1,29 @@ +import { expect, vi } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe7ArgumentsContract as contract } from "../../generated/contracts/facts.f13-transactions-have-limits.probe-7-arguments.contract.js"; +import { schedulingBackend } from "./scheduler-composition.js"; +import { argumentsCase, type SchedulerWorld } from "./scheduler-cases.js"; +const anchor = specTest({ + id: testAnchorId("test:facts.f13-transactions-have-limits.probe-7-arguments"), + verifies: ref("spec:facts.f13-transactions-have-limits.probe-7-arguments"), +}); +void anchor; +vi.setConfig({ testTimeout: 300000 }); +bindExample(contract, (): SchedulerWorld => ({}), { + "the production composition with temporary scheduler functions": async ( + world, + ) => { + Object.assign(world, await schedulingBackend()); + }, + "the native backend exercises arguments": async (world) => { + world.result = await argumentsCase(world.backend!); + }, + "the observation is {result}": (world, { result }) => { + expect(world.result).toBe(result); + }, +}); diff --git a/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts new file mode 100644 index 0000000..7e52830 --- /dev/null +++ b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts @@ -0,0 +1,32 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe7ScanContract as contract } from "../../generated/contracts/facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.contract.js"; +import { schedulingBackend } from "./scheduler-composition.js"; +import { scanCase, type SchedulerWorld } from "./scheduler-cases.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-scan", + ), + verifies: ref( + "spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-scan", + ), +}); +void anchor; +bindExample(contract, (): SchedulerWorld => ({}), { + "the production composition with temporary scheduler functions": async ( + world, + ) => { + Object.assign(world, await schedulingBackend()); + }, + "the native backend exercises scan": async (world) => { + world.result = await scanCase(world.backend!); + }, + "the observation is {result}": (world, { result }) => { + expect(world.result).toBe(result); + }, +}); diff --git a/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts new file mode 100644 index 0000000..aa8f7e1 --- /dev/null +++ b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts @@ -0,0 +1,32 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe7StatesContract as contract } from "../../generated/contracts/facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-states.contract.js"; +import { schedulingBackend } from "./scheduler-composition.js"; +import { statesCase, type SchedulerWorld } from "./scheduler-cases.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-states", + ), + verifies: ref( + "spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-states", + ), +}); +void anchor; +bindExample(contract, (): SchedulerWorld => ({}), { + "the production composition with temporary scheduler functions": async ( + world, + ) => { + Object.assign(world, await schedulingBackend()); + }, + "the native backend exercises states": async (world) => { + world.result = await statesCase(world.backend!); + }, + "the observation is {result}": (world, { result }) => { + expect(world.result).toBe(result); + }, +}); diff --git a/tests/native/scheduler-cases.ts b/tests/native/scheduler-cases.ts new file mode 100644 index 0000000..e188ff8 --- /dev/null +++ b/tests/native/scheduler-cases.ts @@ -0,0 +1,347 @@ +import { expect } from "vitest"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import type { Value } from "convex/values"; +import type { Backend } from "../../harness/backend.js"; +import { measure } from "../../harness/native.js"; +import { paceAfterWrite } from "../../harness/wait.js"; +import { archiveEntries } from "./snapshot-archive.js"; +import { + initialize, + schedulerRows, + dataRows, + record, +} from "./scheduler-composition.js"; +import { + grant, + grantedPermissions, + subject, +} from "./first-experiment-steps.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { api } from "../../example/convex/_generated/api.js"; +export interface SchedulerWorld { + backend?: Backend; + directory?: string; + result?: string; +} +export async function statesCase(backend: Backend) { + await initialize(backend, Date.now() + 3600000); + const before = await schedulerRows(backend); + record("scheduler states", before); + for (const rows of Object.values(before)) { + for (const row of rows) { + expect(typeof row.name).toBe("string"); + expect(Array.isArray(row.args)).toBe(true); + expect(typeof row.scheduledTime).toBe("number"); + const state = row.state as { kind: string; error?: string }; + if (["success", "failed", "canceled"].includes(state.kind)) + expect(typeof row.completedTime).toBe("number"); + else expect(row.completedTime).toBeUndefined(); + if (state.kind === "failed") + expect(state.error).toContain("reaction refused"); + } + } + const intervalStart = Date.now(); + await sleep(1500); + const after = await schedulerRows(backend); + record("retention observation", { + intervalStart, + observedAt: Date.now(), + rows: after, + }); + expect(after).toEqual(before); + return "all five states readable"; +} +interface Attempt { + size: number; + count: number; + character: string; + parts: number; + returned?: number; + error: string | null; + stage: string; + utf8Bytes: number; + jsonBytes: number; + countedBytes?: number; + trim: number; +} +export async function argumentsCase(backend: Backend) { + const attempts: Attempt[] = []; + async function attempt( + size: number, + count = 1, + character = "a", + parts = 1, + trim = 0, + ) { + const value = character.repeat(size); + const payload = + parts === 1 + ? value + : Array.from({ length: parts }, (_, i) => + i === parts - 1 ? value.slice(0, value.length - trim) : value, + ); + const input = { size, count, character, parts, trim }; + const observed: Attempt = { + ...input, + error: null, + stage: "committed", + utf8Bytes: + (Buffer.byteLength(value) * parts - + Buffer.byteLength(character) * trim) * + count, + jsonBytes: Buffer.byteLength(JSON.stringify({ payload })) * count, + }; + const mark = await backend.admin.logMark(); + try { + const result = (await backend.admin.run( + "scheduling:argumentsLimit", + input, + )) as { + returned: number; + error: string | null; + metrics: { scheduledFunctionArgsBytes: { used: number } }; + }; + Object.assign(observed, { + returned: result.returned, + countedBytes: result.metrics.scheduledFunctionArgsBytes.used, + error: result.error, + stage: result.error === null ? "committed" : "runAfter", + }); + record("argument metrics", result.metrics); + } catch (error) { + observed.error = String(error); + observed.stage = "commit or uncatchable syscall"; + } + const completions = await backend.admin.completionsSince(mark, (rows) => + rows.some((row) => row.identifier === "scheduling:argumentsLimit"), + ); + record("argument completion records", completions as unknown as Value); + attempts.push(observed); + measure("argument attempt", { ...observed }); + await paceAfterWrite(Math.min(observed.utf8Bytes + 1024, 17 * 1024 * 1024)); + return observed; + } + for (const boundary of [ + 4 * 1024 * 1024, + 8000000, + 8 * 1024 * 1024, + 16 * 1024 * 1024, + ]) + for (const delta of [-64, 64]) await attempt(boundary + delta); + await attempt(4 * 1024 * 1024 - 16); + await attempt(4 * 1024 * 1024); + await attempt(2 * 1024 * 1024 - 32, 1, "é"); + await attempt(2 * 1024 * 1024 + 32, 1, "é"); + await attempt(4 * 1024 - 4, 1, "a", 1024); + await attempt(4 * 1024, 1, "a", 1024); + for (const boundary of [8000000, 8 * 1024 * 1024, 16 * 1024 * 1024]) + for (const delta of [-256, 256]) + await attempt(Math.floor((boundary + delta) / 8), 8); + for (const character of ["a", "é", "😀", "\n", '"']) + for (const parts of [1, 2, 1024]) await attempt(3, 1, character, parts); + const exact = await attempt(16 * 1024 * 1024 - 14); + const over = await attempt(16 * 1024 * 1024 - 13); + const multibyte = await attempt((16 * 1024 * 1024 - 14) / 2, 1, "é"); + const multibyteOver = await attempt((16 * 1024 * 1024 - 14) / 2 + 1, 1, "é"); + const many = await attempt(16382, 1, "a", 1024, 14); + const manyOver = await attempt(16382, 1, "a", 1024, 13); + const sum = await attempt(2 * 1024 * 1024 - 14, 8); + const sumOver = await attempt(2 * 1024 * 1024 - 13, 8); + record( + "argument boundary summary", + attempts.map((a) => ({ ...a })), + ); + for (const a of attempts) { + const perCall = + a.utf8Bytes / a.count + 14 + (a.parts === 1 ? 0 : 2 * a.parts); + const accepted = Math.min(a.count, Math.floor(16777216 / perCall)); + expect(a.stage, JSON.stringify(a)).toBe( + accepted === a.count ? "committed" : "runAfter", + ); + expect(a.returned, JSON.stringify(a)).toBe(accepted); + if (accepted < a.count) + expect(a.error).toContain( + "Too large total size of the arguments of scheduled functions from this mutation (limit: 16777216 bytes)", + ); + else expect(a.error).toBeNull(); + } + for (const a of [exact, multibyte, many, sum]) + expect(a.countedBytes).toBe(16777216); + for (const a of [over, multibyteOver, manyOver, sumOver]) + expect(a.error).not.toBeNull(); + return "16 MiB summed Convex value bytes per mutation; 4 MiB per call warns"; +} + +export async function scanCase(backend: Backend) { + await backend.admin.writeTable("schedulerData", { + insert: { value: "local", held: true }, + }); + await backend.admin.run("scheduling:states", { + label: "local", + due: Date.now() + 3600000, + }); + await expect + .poll( + async () => (await backend.admin.readTable("schedulerEffects")).length, + ) + .toBe(1); + await expect + .poll( + async () => + (await backend.admin.readTable("_scheduled_functions")).filter( + (r) => (r.state as { kind: string }).kind === "failed", + ).length, + ) + .toBe(1); + async function scan(outstanding: number) { + const mark = await backend.admin.logMark(); + let answer: Value = null; + let error: string | null = null; + try { + answer = await backend.admin.run("scheduling:scan"); + } catch (caught) { + error = String(caught); + } + record("scan observation", { + totalRows: outstanding, + outstanding: outstanding - 3, + pending: outstanding - 4, + answer, + error, + }); + const logs = await backend.admin.completionsSince(mark, (logs) => + logs.some((l) => l.identifier.includes("scheduling:scan")), + ); + measure( + "scan completion usage", + logs.map((l) => ({ + identifier: l.identifier, + usageStats: l.usageStats, + error: l.error, + })), + ); + return { + answer: answer as { + rows: Value[]; + before: { documentsRead: { used: number } }; + after: { documentsRead: { used: number } }; + } | null, + error, + }; + } + const first = await scan(5); + expect(first.answer?.rows).toHaveLength(1); + let count = 5; + const observations = []; + for (const target of [1000, 4090, 4091, 4092, 4096, 32000, 32001]) { + while (count < target) { + const batch = Math.min(500, target - count); + await backend.admin.run("scheduling:populate", { count: batch }); + count += batch; + } + const observed = await scan(count); + observations.push({ count, ...observed }); + if (target === 4091 || target === 4092) { + let rows: Value = null; + let error: string | null = null; + try { + rows = await backend.admin.run("scheduling:scanPlain"); + } catch (caught) { + error = String(caught); + } + record("plain scan observation", { + totalRows: target, + outstanding: target - 3, + rows, + error, + }); + if (target === 4091) { + expect(error).toBeNull(); + expect(rows).toHaveLength(1); + } else + expect(error).toContain( + "Too many reads in a single function execution (limit: 4096)", + ); + } + } + const actualRows = (await backend.admin.readTable("_scheduled_functions")) + .length; + measure("scheduler rows after scan boundary", { + total: actualRows, + outstanding: actualRows - 3, + }); + expect(actualRows).toBe(32001); + for (const observed of observations) { + if (observed.count <= 4091) { + expect(observed.error).toBeNull(); + expect(observed.answer?.rows).toHaveLength(1); + expect( + observed.answer!.after.documentsRead.used - + observed.answer!.before.documentsRead.used, + ).toBe(0); + } else if (observed.count <= 4096) + expect(observed.error).toContain( + "Too many reads in a single function execution (limit: 4096)", + ); + else + expect(observed.error).toMatch( + /Too many reads in a single function execution \(limit: 4096\)|too many system operations/, + ); + } + return "4091 rows scan and 4092 exceed 4096 reads; zero document reads counted"; +} + +async function productionData(backend: Backend) { + await grant(backend, subject, grantedPermissions); + await backend.admin.run("readModels:activate", { + readModel: "orderSummary", + startedBy: { kind: "operator", id: "native-test" }, + }); + const client = ordinaryClient(backend.url, { + token: await backend.issuer.token(subject), + }); + await client.mutation(api.receiving.receiveStock, { + tenantId: "t-1", + input: { items: [{ stockItemId: "sku", quantity: 3 }] }, + }); + await client.mutation(api.ordering.placeOrder, { + tenantId: "t-1", + requestKey: "order", + input: { + orderId: "order", + lines: [{ stockItemId: "sku", quantity: 1, unitPrice: 10 }], + }, + }); +} +export async function exportData( + backend: Backend, + directory: string, + due: number, +) { + await productionData(backend); + await initialize(backend, due); + await backend.admin.setEnvironment({ SCHEDULER_VALUE: "exported" }); + record("source environment", await backend.admin.environment()); + const data = await dataRows(backend); + const schedules = await schedulerRows(backend); + record("exported data", data); + record("exported schedules", schedules); + const path = join(directory, "snapshot.zip"); + await backend.admin.exportSnapshot(path); + const entries = await archiveEntries(path); + measure("archive entries", entries); + for (const entry of [ + "receipts/documents.jsonl", + "orderSummaries/documents.jsonl", + "_components/orders/streams/documents.jsonl", + "_components/orders/events/documents.jsonl", + "_components/inventory/streams/documents.jsonl", + "_components/inventory/events/documents.jsonl", + ]) + expect(entries).toContain(entry); + expect(entries.some((e) => e.includes("_scheduled_functions"))).toBe(false); + expect(await dataRows(backend)).toEqual(data); + expect(await schedulerRows(backend)).toEqual(schedules); + return { data, schedules, path }; +} diff --git a/tests/native/scheduler-composition.ts b/tests/native/scheduler-composition.ts new file mode 100644 index 0000000..925882b --- /dev/null +++ b/tests/native/scheduler-composition.ts @@ -0,0 +1,132 @@ +import { + cp, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { onTestFinished, expect } from "vitest"; +import type { Backend } from "../../harness/backend.js"; +import { productionBackend, measure } from "../../harness/native.js"; +import { convexToJson, type Value } from "convex/values"; +const root = join(import.meta.dirname, "../.."); +export const scopes = [undefined, "orders", "inventory"] as const; +export const options = (component: string | undefined) => + component === undefined ? {} : { component }; +export async function schedulingBackend() { + const directory = await mkdtemp(join(tmpdir(), "scheduler-composition-")); + onTestFinished(() => rm(directory, { recursive: true, force: true })); + for (const path of ["example", "src", "package.json"]) + await cp(join(root, path), join(directory, path), { recursive: true }); + await symlink(join(root, "node_modules"), join(directory, "node_modules")); + const functions = await readFile( + join(import.meta.dirname, "scheduler-functions.ts"), + "utf8", + ); + const schemas: Record = {}; + for (const component of scopes) { + const path = join(directory, "example/convex", component ?? ""); + await writeFile(join(path, "scheduling.ts"), functions); + let schema = await readFile(join(path, "schema.ts"), "utf8"); + schema = 'import { v } from "convex/values";\n' + schema; + if (!schema.includes("defineTable")) + schema = schema.replace( + "{ defineSchema }", + "{ defineSchema, defineTable }", + ); + schema = schema.replace( + "defineSchema({", + 'defineSchema({ schedulerData: defineTable({ value: v.string(), held: v.boolean() }), schedulerReferences: defineTable({ dispatchId: v.id("_scheduled_functions"), label: v.string() }), schedulerEffects: defineTable({ label: v.string(), value: v.string() }),', + ); + await writeFile(join(path, "schema.ts"), schema); + schemas[component ?? "parent"] = createHash("sha256") + .update(schema) + .digest("hex"); + } + const backend = await productionBackend(); + const deployed = await backend.admin.deployTemporary( + join(directory, "example"), + ); + measure("temporary deployment", { + baseComposition: "production", + addedModule: "scheduling", + scopes: scopes.map((s) => s ?? "parent"), + functionSha256: createHash("sha256").update(functions).digest("hex"), + schemas, + ...deployed, + }); + expect(backend.facts().composition).toBeNull(); + return { backend, directory }; +} +export async function initialize( + backend: Backend, + due: number, + label = "exported", +) { + for (const component of scopes) { + await backend.admin.writeTable( + "schedulerData", + { insert: { value: "exported", held: true } }, + options(component), + ); + await backend.admin.run( + "scheduling:states", + { label, due }, + options(component), + ); + } + for (const component of scopes) + await expect + .poll( + async () => + ( + await backend.admin.readTable( + "_scheduled_functions", + options(component), + ) + ) + .map((row) => (row.state as { kind: string }).kind) + .sort(), + { timeout: 15000 }, + ) + .toEqual(["canceled", "failed", "inProgress", "pending", "success"]); +} +export async function schedulerRows(backend: Backend) { + const result: Record[]> = {}; + for (const component of scopes) + result[component ?? "parent"] = await backend.admin.readTable( + "_scheduled_functions", + options(component), + ); + return result; +} +export async function dataRows(backend: Backend) { + const result: Record[]> = {}; + for (const component of scopes) + for (const table of component === undefined + ? [ + "receipts", + "grants", + "generations", + "orderSummaries", + "schedulerData", + "schedulerReferences", + "schedulerEffects", + ] + : [ + "streams", + "events", + "schedulerData", + "schedulerReferences", + "schedulerEffects", + ]) + result[`${component ?? "parent"}/${table}`] = + await backend.admin.readTable(table, options(component)); + return result; +} +export const record = (name: string, value: Value) => + measure(name, convexToJson(value)); diff --git a/tests/native/scheduler-functions.ts b/tests/native/scheduler-functions.ts new file mode 100644 index 0000000..c8dd5ab --- /dev/null +++ b/tests/native/scheduler-functions.ts @@ -0,0 +1,159 @@ +import { + internalMutationGeneric as mutation, + internalQueryGeneric as query, + internalActionGeneric as action, + makeFunctionReference, +} from "convex/server"; +import { v } from "convex/values"; +const mutationRef = (name: string) => + makeFunctionReference<"mutation">(`scheduling:${name}`); +export const reaction = mutation({ + args: { label: v.string(), fail: v.optional(v.boolean()) }, + handler: async (ctx, args) => { + if (args.fail) throw new Error(`reaction refused: ${args.label}`); + const data = await ctx.db.query("schedulerData").first(); + await ctx.db.insert("schedulerEffects", { + label: args.label, + value: data?.value ?? "absent", + }); + }, +}); +export const held = action({ + args: { label: v.string() }, + handler: async (ctx) => { + while ( + await ctx.runQuery( + makeFunctionReference<"query">("scheduling:isHeld"), + {}, + ) + ) + await new Promise((resolve) => setTimeout(resolve, 200)); + return null; + }, +}); +export const isHeld = query({ + args: {}, + handler: async (ctx) => + (await ctx.db.query("schedulerData").first())?.held === true, +}); +export const states = mutation({ + args: { label: v.string(), due: v.number() }, + handler: async (ctx, { label, due }) => { + const pending = await ctx.scheduler.runAt(due, mutationRef("reaction"), { + label, + }); + await ctx.db.insert("schedulerReferences", { dispatchId: pending, label }); + await ctx.scheduler.runAfter(0, mutationRef("reaction"), { + label: `${label}-success`, + }); + await ctx.scheduler.runAfter(0, mutationRef("reaction"), { + label: `${label}-failed`, + fail: true, + }); + const canceled = await ctx.scheduler.runAfter( + 3600000, + mutationRef("reaction"), + { label: `${label}-canceled` }, + ); + await ctx.scheduler.cancel(canceled); + await ctx.scheduler.runAfter( + 0, + makeFunctionReference<"action">("scheduling:held"), + { label }, + ); + return pending; + }, +}); +export const reference = mutation({ + args: { id: v.id("_scheduled_functions"), cancel: v.boolean() }, + handler: async (ctx, { id, cancel }) => { + await ctx.db.insert("schedulerReferences", { + dispatchId: id, + label: "copied", + }); + const row = await ctx.db.system.get(id); + if (cancel) await ctx.scheduler.cancel(id); + return row; + }, +}); +export const consume = mutation({ + args: { payload: v.any() }, + handler: () => null, +}); +export const argumentsLimit = mutation({ + args: { + size: v.number(), + count: v.number(), + character: v.string(), + parts: v.number(), + trim: v.optional(v.number()), + }, + handler: async (ctx, { size, count, character, parts, trim = 0 }) => { + const value = character.repeat(size); + const payload = + parts === 1 + ? value + : Array.from({ length: parts }, (_, i) => + i === parts - 1 ? value.slice(0, value.length - trim) : value, + ); + let returned = 0; + try { + for (; returned < count; returned++) + await ctx.scheduler.runAfter(3600000, mutationRef("consume"), { + payload, + }); + return { + returned, + error: null, + metrics: await ctx.meta.getTransactionMetrics(), + }; + } catch (error) { + return { + returned, + error: String(error), + metrics: await ctx.meta.getTransactionMetrics(), + }; + } + }, +}); +export const populate = mutation({ + args: { count: v.number() }, + handler: async (ctx, { count }) => { + for (let i = 0; i < count; i++) + await ctx.scheduler.runAfter( + 3600000, + mutationRef(i % 2 ? "consume" : "reaction"), + i % 2 ? { payload: "unrelated" } : { label: "unrelated" }, + ); + }, +}); +export const scan = query({ + args: {}, + handler: async (ctx) => { + const before = await ctx.meta.getTransactionMetrics(); + const rows = await ctx.db.system + .query("_scheduled_functions") + .filter((q) => + q.and( + q.eq(q.field("name"), "scheduling.js:reaction"), + q.eq(q.field("state.kind"), "failed"), + ), + ) + .collect(); + return { rows, before, after: await ctx.meta.getTransactionMetrics() }; + }, +}); + +export const scanPlain = query({ + args: {}, + handler: async (ctx) => + await ctx.db.system + .query("_scheduled_functions") + .filter((q) => + q.and( + q.eq(q.field("name"), "scheduling.js:reaction"), + q.eq(q.field("state.kind"), "failed"), + ), + ) + .collect(), +}); diff --git a/tests/native/scheduler-restore.ts b/tests/native/scheduler-restore.ts new file mode 100644 index 0000000..556a436 --- /dev/null +++ b/tests/native/scheduler-restore.ts @@ -0,0 +1,189 @@ +import { expect } from "vitest"; +import type { Backend } from "../../harness/backend.js"; +import { measure } from "../../harness/native.js"; +import { exportData } from "./scheduler-cases.js"; +import { + schedulingBackend, + schedulerRows, + dataRows, + scopes, + options, + record, +} from "./scheduler-composition.js"; +export async function freshImport(backend: Backend, directory: string) { + const exported = await exportData(backend, directory, Date.now() + 3600000); + const { backend: fresh } = await schedulingBackend(); + await fresh.admin.setEnvironment({ SCHEDULER_VALUE: "destination" }); + const environment = await fresh.admin.environment(); + await fresh.admin.replaceSnapshot(exported.path); + const restored = await dataRows(fresh); + record("fresh restored data", restored); + expect(restored).toEqual(exported.data); + const schedules = await schedulerRows(fresh); + record("fresh scheduler tables", schedules); + for (const rows of Object.values(schedules)) expect(rows).toEqual([]); + expect(await fresh.admin.environment()).toEqual(environment); + record("fresh environment after import", await fresh.admin.environment()); + measure("fresh environment preserved", true); + for (const component of scopes) { + const stored = await fresh.admin.readTable( + "schedulerReferences", + options(component), + ); + expect(stored).toHaveLength(1); + const id = stored[0]!.dispatchId!; + let read: unknown; + let error: string | null = null; + try { + read = await fresh.admin.run( + "scheduling:reference", + { id, cancel: true }, + options(component), + ); + } catch (caught) { + error = String(caught); + } + record("fresh scheduler reference", { + component: component ?? "parent", + id, + error, + row: read === undefined ? null : (read as null), + }); + expect(error).toBeNull(); + expect(read).toBeNull(); + const copies = await fresh.admin.readTable( + "schedulerReferences", + options(component), + ); + expect(copies).toHaveLength(2); + expect(copies.every((row) => row.dispatchId === id)).toBe(true); + expect( + await fresh.admin.readTable("_scheduled_functions", options(component)), + ).toEqual([]); + } + return "documents preserved and schedules absent"; +} +export async function inPlaceImport(backend: Backend, directory: string) { + const due = Date.now() + 45000; + const exported = await exportData(backend, directory, due); + for (const component of scopes) { + const rows = await backend.admin.readTable( + "schedulerData", + options(component), + ); + await backend.admin.writeTable( + "schedulerData", + { patch: String(rows[0]!._id), fields: { value: "after-export" } }, + options(component), + ); + await backend.admin.run( + "scheduling:states", + { label: "after-export", due }, + options(component), + ); + } + for (const component of scopes) + await expect + .poll( + async () => + ( + await backend.admin.readTable( + "_scheduled_functions", + options(component), + ) + ) + .map((row) => (row.state as { kind: string }).kind) + .sort(), + { timeout: 15000 }, + ) + .toEqual([ + "canceled", + "canceled", + "failed", + "failed", + "inProgress", + "inProgress", + "pending", + "pending", + "success", + "success", + ]); + await backend.admin.setEnvironment({ SCHEDULER_VALUE: "after-export" }); + const environment = await backend.admin.environment(); + const before = await schedulerRows(backend); + record("in-place scheduler tables before import", before); + await backend.admin.replaceSnapshot(exported.path); + expect(Date.now()).toBeLessThan(due); + const after = await schedulerRows(backend); + record("in-place scheduler tables after import", after); + expect(after).toEqual(before); + const restored = await dataRows(backend); + record("in-place restored data", restored); + expect(restored).toEqual(exported.data); + expect(await backend.admin.environment()).toEqual(environment); + record( + "in-place environment after import", + await backend.admin.environment(), + ); + measure("in-place environment preserved", true); + for (const component of scopes) { + const id = + exported.data[`${component ?? "parent"}/schedulerReferences`]![0]! + .dispatchId!; + const found = await backend.admin.run( + "scheduling:reference", + { id, cancel: true }, + options(component), + ); + record("in-place scheduler reference", { + component: component ?? "parent", + id, + row: found, + }); + expect(found).toEqual( + before[component ?? "parent"]!.find((row) => row._id === id), + ); + const canceled = ( + await backend.admin.readTable("_scheduled_functions", options(component)) + ).find((row) => row._id === id); + record("canceled restored reference", { + component: component ?? "parent", + row: canceled!, + }); + expect(canceled?.state).toEqual({ kind: "canceled" }); + } + for (const component of scopes) { + await expect + .poll( + async () => + ( + await backend.admin.readTable( + "schedulerEffects", + options(component), + ) + ).filter((row) => row.label === "after-export"), + { timeout: 55000, interval: 200 }, + ) + .toEqual([ + expect.objectContaining({ label: "after-export", value: "exported" }), + ]); + const effects = await backend.admin.readTable( + "schedulerEffects", + options(component), + ); + record("kept reactions after their scheduled time", { + component: component ?? "parent", + due, + observedAt: Date.now(), + effects, + }); + const kept = effects.find((row) => row.label === "after-export"); + expect(Number(kept?._creationTime)).toBeGreaterThanOrEqual(due); + expect(effects.filter((row) => row.label === "exported")).toEqual([]); + } + record( + "in-place scheduler tables after execution", + await schedulerRows(backend), + ); + return "documents replaced and schedules preserved"; +} diff --git a/tests/native/snapshot-archive.ts b/tests/native/snapshot-archive.ts new file mode 100644 index 0000000..8ec8de9 --- /dev/null +++ b/tests/native/snapshot-archive.ts @@ -0,0 +1,34 @@ +import { readFile } from "node:fs/promises"; +// ZIP central-directory names need no decompressor or external executable. +export async function archiveEntries(path: string): Promise { + const bytes = await readFile(path); + let end = bytes.length - 22; + for (; end >= Math.max(0, bytes.length - 65557); end--) { + if ( + bytes.readUInt32LE(end) === 0x06054b50 && + end + 22 + bytes.readUInt16LE(end + 20) === bytes.length + ) + break; + } + if (end < 0 || bytes.readUInt32LE(end) !== 0x06054b50) + throw new Error("ZIP end record missing"); + const count = bytes.readUInt16LE(end + 10); + let cursor = bytes.readUInt32LE(end + 16); + if (count === 65535 || cursor === 0xffffffff) + throw new Error("ZIP64 directory not supported"); + const entries: string[] = []; + for (let i = 0; i < count; i++) { + if (bytes.readUInt32LE(cursor) !== 0x02014b50) + throw new Error("ZIP directory entry missing"); + const length = bytes.readUInt16LE(cursor + 28); + entries.push( + bytes.subarray(cursor + 46, cursor + 46 + length).toString("utf8"), + ); + cursor += + 46 + + length + + bytes.readUInt16LE(cursor + 30) + + bytes.readUInt16LE(cursor + 32); + } + return entries; +} diff --git a/tests/native/snapshot-replacement.test.ts b/tests/native/snapshot-replacement.test.ts index 21aa61d..00500cd 100644 --- a/tests/native/snapshot-replacement.test.ts +++ b/tests/native/snapshot-replacement.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { convexToJson } from "convex/values"; import { expect, onTestFinished, test } from "vitest"; import type { Backend } from "../../harness/backend.js"; -import { runChild } from "../../harness/child.js"; +import { archiveEntries } from "./snapshot-archive.js"; import { fixtureBackend, measure } from "../../harness/native.js"; import { deploySnapshotFixture } from "../../harness/snapshot.js"; @@ -63,7 +63,12 @@ test("native: snapshot replacement restores parent and context data and preserve await schedulingComposition(directory); const source = await fixtureBackend(); const deployed = await deploySnapshotFixture(source, directory, signal); - measure("fixture scheduling deployment command", { ...deployed }); + measure("fixture scheduling deployment command", { + baseComposition: "fixture", + addedModule: "snapshotSchedule:schedule", + ...deployed, + }); + expect(source.facts().composition).toBeNull(); await createDocuments(source, "exported"); await source.admin.setEnvironment({ SNAPSHOT_VALUE: "exported" }); await source.admin.run("snapshotSchedule:schedule"); @@ -80,10 +85,7 @@ test("native: snapshot replacement restores parent and context data and preserve measure("snapshot source", convexToJson(before)); const path = join(directory, "snapshot.zip"); await source.admin.exportSnapshot(path); - const entries = await runChild("unzip", "unzip", ["-Z1", path], { - timeoutMs: 10000, - signal, - }); + const entries = await archiveEntries(path); measure("snapshot archive entries", entries); for (const entry of [ "markers/documents.jsonl", @@ -92,7 +94,9 @@ test("native: snapshot replacement restores parent and context data and preserve "_components/depot/events/documents.jsonl", ]) expect(entries).toContain(entry); - expect(entries).not.toContain("_scheduled_functions"); + expect(entries.some((entry) => entry.includes("_scheduled_functions"))).toBe( + false, + ); expect(await observe(source)).toEqual(before); const fresh = await fixtureBackend(); From 95fe4e4db406eac575cde2db9029b9cbe2048e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:09:58 +0200 Subject: [PATCH 011/155] Probe commit timestamps, call budgets and page byte thresholds --- design/README.md | 21 ++- .../first-experiment.order-of-1-line.sdp.md | 2 +- .../first-experiment.order-of-10-lines.sdp.md | 2 +- ...first-experiment.order-of-max-lines.sdp.md | 2 +- ....committed-state-read-through-query.sdp.md | 2 +- ...read-models.committed-state-visible.sdp.md | 2 +- design/specs/context/queries.sdp.md | 1 + ...tamps.probe-8-parent-and-components.sdp.md | 30 ++++ .../specs/facts/f18-commit-timestamps.sdp.md | 34 +++++ ...me-budgets.probe-11-component-calls.sdp.md | 30 ++++ ...re-time-budgets.probe-9-nested-time.sdp.md | 29 ++++ ...f19-nested-calls-share-time-budgets.sdp.md | 34 +++++ ...fter-reading.probe-10-oversized-row.sdp.md | 29 ++++ ...fter-reading.probe-10-stream-budget.sdp.md | 29 ++++ ...gination-checks-bytes-after-reading.sdp.md | 32 ++++ design/specs/facts/fact-ledger.sdp.md | 1 + design/specs/facts/probe-plan.sdp.md | 8 + design/specs/foundation.pack.sdp.md | 8 + fixture/convex/_generated/api.d.ts | 9 ++ fixture/convex/annex/_generated/api.ts | 8 + fixture/convex/annex/_generated/component.ts | 52 +++++++ fixture/convex/annex/bytePage.ts | 56 +++++++ fixture/convex/annex/callBudget.ts | 16 ++ fixture/convex/annex/scheduledRows.ts | 15 ++ fixture/convex/annex/schema.ts | 4 + fixture/convex/annex/timestamps.ts | 31 ++++ fixture/convex/bytePage.ts | 9 ++ fixture/convex/callBudget.ts | 54 +++++++ fixture/convex/convex.config.ts | 1 + fixture/convex/scheduledRows.ts | 23 +++ fixture/convex/schema.ts | 4 + fixture/convex/timestamps.ts | 69 +++++++++ tests/native/call-budget.ts | 133 +++++++++++++++++ ...amps.probe-8-parent-and-components.test.ts | 140 ++++++++++++++++++ ...e-budgets.probe-11-component-calls.test.ts | 35 +++++ ...e-time-budgets.probe-9-nested-time.test.ts | 35 +++++ ...ter-reading.probe-10-oversized-row.test.ts | 92 ++++++++++++ ...ter-reading.probe-10-stream-budget.test.ts | 74 +++++++++ tests/native/first-experiment-steps.ts | 21 ++- ...committed-state-read-through-query.test.ts | 18 +-- ...ead-models.committed-state-visible.test.ts | 18 +-- tests/native/scheduled-rows.test.ts | 24 +++ tests/native/scheduled-rows.ts | 13 ++ 43 files changed, 1205 insertions(+), 45 deletions(-) create mode 100644 design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md create mode 100644 design/specs/facts/f18-commit-timestamps.sdp.md create mode 100644 design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md create mode 100644 design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md create mode 100644 design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md create mode 100644 design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md create mode 100644 design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md create mode 100644 design/specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md create mode 100644 fixture/convex/annex/bytePage.ts create mode 100644 fixture/convex/annex/callBudget.ts create mode 100644 fixture/convex/annex/scheduledRows.ts create mode 100644 fixture/convex/annex/timestamps.ts create mode 100644 fixture/convex/bytePage.ts create mode 100644 fixture/convex/callBudget.ts create mode 100644 fixture/convex/scheduledRows.ts create mode 100644 fixture/convex/timestamps.ts create mode 100644 tests/native/call-budget.ts create mode 100644 tests/native/f18-commit-timestamps.probe-8-parent-and-components.test.ts create mode 100644 tests/native/f19-nested-calls-share-time-budgets.probe-11-component-calls.test.ts create mode 100644 tests/native/f19-nested-calls-share-time-budgets.probe-9-nested-time.test.ts create mode 100644 tests/native/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.test.ts create mode 100644 tests/native/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.test.ts create mode 100644 tests/native/scheduled-rows.test.ts create mode 100644 tests/native/scheduled-rows.ts diff --git a/design/README.md b/design/README.md index 96361e1..e7b561d 100644 --- a/design/README.md +++ b/design/README.md @@ -2,7 +2,7 @@ ## What this is -The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 168 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. +The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 210 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. The thesis, in the doc's words. Ownership follows bounded contexts. Atomicity follows the business operation. Asynchrony follows a concrete need to defer work. The default operation is one Convex mutation that authorizes a command, makes the domain decision, saves current state with its events, updates essential read models and records the outcome. A second transaction exists only to wait, to spread load, or to reach an external system. The design succeeds when adding domain sophistication does not add infrastructure. @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -202 specs · 5 packs · 101 anchors → 308 nodes · 1097 edges (0 errors, 0 warnings) +210 specs · 5 packs · 106 anchors → 321 nodes · 1127 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -82,7 +82,7 @@ Detail follows the build (decision method, rule 4). Full means interfaces, TypeS | Layer | Families | Pack | Specs | |---|---|---|---| -| all | `platform`, `laws`, `decisions`, `facts` | [`foundation`](specs/foundation.pack.sdp.md) | 75 | +| all | `platform`, `laws`, `decisions`, `facts` | [`foundation`](specs/foundation.pack.sdp.md) | 87 | | 0 | `kernel` | [`kernel-and-context`](specs/kernel-and-context.pack.sdp.md) | 7 | | 1 | `context`, `command` | [`kernel-and-context`](specs/kernel-and-context.pack.sdp.md) (19 with the kernel), [`command-pipeline`](specs/command-pipeline.pack.sdp.md) | 12 + 22 | | 2 | `application`, `operations`, `constraints` | [`application`](specs/application.pack.sdp.md) | 31 | @@ -116,7 +116,9 @@ The doc numbers its decisions, laws, sources and probes. The plan numbers everyt | `D1` to `D19` | decision, in the doc's order | | `Law 1` to `Law 12` | law | | `F1` to `F17` | fact-ledger row, in table order | +| `F18` to `F20` | commit timestamps, nested-call time budgets and the paginator byte check | | `Probe 1` to `Probe 7` | probe, in list order | +| `Probe 8` to `Probe 11` | commit timestamps, nested-call time, paginator bytes and component calls per mutation | | `S1` to `S15` | source | | `Sc L0-1` to `Sc L6-4`, `Sc ALL-1` | acceptance scenario, by layer then table order | | `OQ1` to `OQ6` | open question of the doc, in bullet order | @@ -272,6 +274,12 @@ Beyond the plan's list, three product decisions the packages surfaced: Probes 1 to 5 ran on a native backend in slice S0 on 2026-10-01. Each result is recorded on its fact and on the Specs that rest on it, and [`probe-plan`](specs/facts/probe-plan.sdp.md) says what stays open: Probe 1 (the client guarantee holds across a restart, a closed client can lose a command, and the HTTP client does not retry, D6), Probe 2 (`ConvexError` data survives a nested mutation and a component boundary, F14, D7), Probe 3 (a component call costs about 2 ms more than a helper call on a local backend, with hosted cost and quota open, F4, D2, D8), Probe 4 (transaction limits are one budget across nested calls and components, F13, D10), Probe 5 (a parent query over a component query stays reactive and pinned pages stay contiguous, and the client hook loses rows after a capped split, F15, D8). Two are pending: Probe 6 (a backfill batch racing a live command; the migrations component, F17, D9) and Probe 7 (what a restore leaves of scheduler, Workpool and Workflow state; the system table's states and retention, F12, F16, D13, D19). +| Tier | Fact | Subject | +|---|---|---| +| Native backend | [`spec:facts.f18-commit-timestamps`](specs/facts/f18-commit-timestamps.sdp.md) | Commit timestamps order committed rows | +| Native backend | [`spec:facts.f19-nested-calls-share-time-budgets`](specs/facts/f19-nested-calls-share-time-budgets.sdp.md) | Nested calls share time budgets | +| Native backend | [`spec:facts.f20-pagination-checks-bytes-after-reading`](specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md) | Pagination checks bytes after reading | + Every open question in the corpus, by Spec, is recipe 20 of the Protocol's catalog, the open-question register. It lists the Specs that hold a blocking question first and reports the totals, so no table is kept here: ```sh @@ -316,8 +324,13 @@ The Layer 2 design added two more for the same reason, and nine examples that ve | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split` | Probe 5: a full first page under a row cap of twice the page size is not split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap` | Probe 5: the same page under a row cap equal to the page size is split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap` | Probe 5: a parent list's page outgrows its row cap | +| none | [`spec:facts.f18-commit-timestamps.probe-8-parent-and-components`](specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md) | Commit timestamps in the parent and two components | +| none | [`spec:facts.f19-nested-calls-share-time-budgets.probe-11-component-calls`](specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md) | Component calls per mutation | +| none | [`spec:facts.f19-nested-calls-share-time-budgets.probe-9-nested-time`](specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md) | Nested calls reach a time boundary | +| none | [`spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-oversized-row`](specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md) | A page keeps the row that crosses its byte bound | +| none | [`spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-stream-budget`](specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md) | A page includes stored system fields in its byte check | -Package A therefore has 79 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. +Package A therefore has 87 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. ## Integration notes diff --git a/design/specs/application/first-experiment.order-of-1-line.sdp.md b/design/specs/application/first-experiment.order-of-1-line.sdp.md index ebcaeab..baef41d 100644 --- a/design/specs/application/first-experiment.order-of-1-line.sdp.md +++ b/design/specs/application/first-experiment.order-of-1-line.sdp.md @@ -29,6 +29,6 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends `PlaceOrder` with one line, reads the backend's function log from a mark before the command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for the command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, and a job scheduled with a delay past the window is not seen, and that the list the read returned already holds the order's summary row. +- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends `PlaceOrder` with one line, reads the backend's function log from a mark before the command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for the command, which committed, and reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, asserting that all three tables are empty, including any function scheduled with a delay, and that the list the read returned already holds the order's summary row. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation for this size. - The test reads the documents and bytes read and written from the completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts six documents written: the receipt, the order's state and event, the stock item's state and event, and the summary row; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-10-lines.sdp.md b/design/specs/application/first-experiment.order-of-10-lines.sdp.md index 8b8b742..afeb002 100644 --- a/design/specs/application/first-experiment.order-of-10-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-10-lines.sdp.md @@ -29,6 +29,6 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` with ten lines over ten stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, and a job scheduled with a delay past the window is not seen, and that the list the read returned already holds the order's summary row. +- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` with ten lines over ten stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, asserting that all three tables are empty, including any function scheduled with a delay, and that the list the read returned already holds the order's summary row. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation carrying the order and then one on the Inventory operation carrying all ten lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts that the ten-line order wrote two documents more per added line than the one-line reference, the added stock item's state and its event, so documents written grew linearly with the lines; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-max-lines.sdp.md b/design/specs/application/first-experiment.order-of-max-lines.sdp.md index cc81287..3890b6a 100644 --- a/design/specs/application/first-experiment.order-of-max-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-max-lines.sdp.md @@ -29,7 +29,7 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the provisional maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, and a job scheduled with a delay past the window is not seen, and that the list the read returned already holds the order's summary row; and it asserts that the completion record's execution time is under the 1 second mutation timeout. +- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the provisional maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, asserting that all three tables are empty, including any function scheduled with a delay, and that the list the read returned already holds the order's summary row; and it asserts that the completion record's execution time is under the 1 second mutation timeout. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation carrying all 100 lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, and asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. - The test then receives stock for 101 more stock items through `ReceiveStock`, sends `PlaceOrder` with one line on each, one line more than the maximum, and asserts the `operationTooLarge` rejection whose details carry 101 items against a maximum of 100, with the receipts, both contexts' stream rows and events and the order summaries unchanged. diff --git a/design/specs/application/read-models.committed-state-read-through-query.sdp.md b/design/specs/application/read-models.committed-state-read-through-query.sdp.md index 439cb46..415cf03 100644 --- a/design/specs/application/read-models.committed-state-read-through-query.sdp.md +++ b/design/specs/application/read-models.committed-state-read-through-query.sdp.md @@ -26,4 +26,4 @@ And the number of workers, jobs or queues that ran is {workers: 0} - Runs in the native tier on the production composition; every test owns its disposable backend. - After grants, the first activation and `ReceiveStock` as setup, the test runs `PlaceOrder` and then, from the same client, calls the parent query `getOrder`, which relays the Orders context's `get`, and asserts that the DTO it returns holds the order's committed state and that its `version` equals the command's returned `versions` entry for the order's stream. -- The test asserts that no scheduled function, cron or action ran for the command by reading the disposable backend's function log from a mark before the command to the `getOrder` read that the same client sends after the command returned, a window an ordinary read closes, and finding no record that a client did not cause; a job scheduled with a delay past that window is not seen. +- The test reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, and asserts that all three tables are empty, including any function scheduled with a delay. diff --git a/design/specs/application/read-models.committed-state-visible.sdp.md b/design/specs/application/read-models.committed-state-visible.sdp.md index a0736a3..45154b5 100644 --- a/design/specs/application/read-models.committed-state-visible.sdp.md +++ b/design/specs/application/read-models.committed-state-visible.sdp.md @@ -27,4 +27,4 @@ And the number of workers, jobs or queues that ran is {workers: 0} - Runs in the native tier on the production composition; every test owns its disposable backend. - After grants, the first activation and `ReceiveStock` as setup, the test subscribes to `listOrderSummaries` with the tenant, the status a placed order has and `{ cursor: null, numItems: 10 }`, runs `PlaceOrder`, and asserts that the next subscription value's page carries the order's summary whose `sourceVersions` equal the command's returned `versions` entry for the order's stream, the one stream the summary is projected from. -- The test asserts that the backend ran no scheduled function, cron or action between the command and the subscription update, by reading the disposable backend's function log from before the command until it holds the command's mutation and the subscription's query, a window an ordinary read after the command closes, and finding no record that a client did not cause; a job scheduled with a delay past that window is not seen. +- The test reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, and asserts that all three tables are empty, including any function scheduled with a delay. diff --git a/design/specs/context/queries.sdp.md b/design/specs/context/queries.sdp.md index a7b8301..a3189f6 100644 --- a/design/specs/context/queries.sdp.md +++ b/design/specs/context/queries.sdp.md @@ -7,6 +7,7 @@ relations: refines: spec:context.context-component decidedBy: spec:decisions.d08-read-models-in-command constrainedBy: + - spec:facts.f20-pagination-checks-bytes-after-reading - spec:laws.law11-tenant-scope-named - spec:laws.law05-authorization-before-execution-and-disclosure - spec:facts.f11-components-have-no-ctx-auth diff --git a/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md b/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md new file mode 100644 index 0000000..491c3fa --- /dev/null +++ b/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md @@ -0,0 +1,30 @@ +--- +id: spec:facts.f18-commit-timestamps.probe-8-parent-and-components +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f18-commit-timestamps + verifies: spec:facts.f18-commit-timestamps +--- +# Commit timestamps in the parent and two components + +Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. + +## Intent + +- outcome: Establish the observed behavior on the pinned local backend. + +```gwt +Given a parent and two components that write commit timestamp placeholders +When three mutations commit in sequence and a query reads their indexed rows +Then each transaction has {shared: true} timestamp across the three tables and later transactions have larger timestamps +And returning a placeholder {returned: "resolves to a bigint"} and scheduling it is {refused: true} +``` + +## Verification — executable + +- Every test owns a disposable native backend. Timings and call counts are recorded, not asserted, and say nothing about a hosted deployment. +- The first native run refuted the expectation that returning a mutation placeholder is refused. The returned value is the committed bigint, which this example binds as observed; the scheduling refusal held. +- The query uses the internal runtime `getSnapshotTs()` through an explicit cast. A compiled type assertion checks that the public `QueryMeta` declaration has no such method. This example does not establish a supported public upper bound or hosted behavior. +- The returned bigint equals the stored row timestamp and exceeds the earlier query upper bound; the failed scheduled call leaves no row and no scheduled function. diff --git a/design/specs/facts/f18-commit-timestamps.sdp.md b/design/specs/facts/f18-commit-timestamps.sdp.md new file mode 100644 index 0000000..158fdcd --- /dev/null +++ b/design/specs/facts/f18-commit-timestamps.sdp.md @@ -0,0 +1,34 @@ +--- +id: spec:facts.f18-commit-timestamps +kind: constraint +altitude: story +readiness: defined +relations: + refines: spec:facts.fact-ledger +--- +# Commit timestamps order committed rows + +F18 · Native backend tier · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. + +## Intent + +- outcome: Record the package declaration and native backend behavior with the limits of the evidence. (F18) + +### Open questions + +- [non-blocking] The internal runtime upper-bound method is absent from the public server declarations. A supported public upper bound, ordering under overlapping transactions and behavior on a hosted deployment are outside the native example; no platform consumer adopts the internal method by this fact. (F18) + +## Constraints + +- statement: On the pinned native backend, a transaction writes the same resolved bigint commit timestamp in the parent and two components, later sequential transactions have larger timestamps, and a top-level mutation may return its placeholder as the resolved bigint; within the mutation a read yields the unresolved placeholder and scheduling that placeholder is refused. (F18) +- flavor: convex-fact +- target: evidence.status:probed +- measurableBy: Compiled tier: `convex` 1.46.0, `node_modules/convex/dist/esm-types/server/database.d.ts:308-318` declares the placeholder and `:199` keeps the schema type on insert, and `node_modules/convex/dist/esm-types/server/meta.d.ts:125-136` declares no snapshot timestamp method; package source `node_modules/convex/src/server/meta.ts:137-160` describes the internal `getSnapshotTs()` upper bound, and `node_modules/convex/src/server/impl/meta_impl.ts:56-59,78,94` supplies it at runtime; `node_modules/convex/src/values/value.ts:98-127` defines unresolved numeric conversion; native backend tier: Probe 8 on the pinned release. (F18) + +## Design + +- nativeOrder: Probe 8 commits three sequential transactions, each writing the parent and two component tables indexed by `commitTs`; each transaction has one timestamp across all three tables and the indexed query returns increasing timestamps. Rows in one transaction tie, so a timestamp alone is not a unique event cursor. (F18, Probe 8) +- nativeUpperBound: A cast exposing the internal runtime `ctx.meta.getSnapshotTs()` returns the same upper bound in the parent and both component queries; every observed row is at or below it and a subsequent mutation's returned timestamp is above it. This is native backend evidence for the internal method, not a compiled public API promise. (F18, Probe 8) +- placeholderBehavior: The writing mutation reads back the same placeholder and numeric conversion throws that it is unresolved. Returning the placeholder from a mutation resolves it to the committed row's bigint. Scheduling it fails with `Field name $commitTs starts with '$', which is reserved.` and rolls back the accompanying write. (F18, Probe 8) +- firstObservation: The expectation that returning a mutation's placeholder would fail was refuted on the pinned native backend. The example binds the observed resolved bigint; refusal to schedule held. (F18, Probe 8) +- schemaBoundary: The fixture uses `v.any()` for its timestamp field and asserts the committed value is a bigint. The pinned writer's insert declaration takes the schema's field type and does not widen an int64 field to accept `CommitTsPlaceholder`. (F18, Probe 8) diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md new file mode 100644 index 0000000..74aa3c3 --- /dev/null +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md @@ -0,0 +1,30 @@ +--- +id: spec:facts.f19-nested-calls-share-time-budgets.probe-11-component-calls +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f19-nested-calls-share-time-budgets + verifies: spec:facts.f19-nested-calls-share-time-budgets +--- +# Component calls per mutation + +Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. + +## Intent + +- outcome: Establish the observed behavior on the pinned local backend. + +```gwt +Given a parent mutation that calls an empty component mutation repeatedly +When the client increases the call count until the backend refuses the mutation +Then the error names {budget: "Your request timed out performing too many system operations."} time +And each empty component call reads {reads: 0} and writes {writes: 0} documents +``` + +## Verification — executable + +- Every test owns a disposable native backend. Timings and call counts are recorded, not asserted, and say nothing about a hosted deployment. +- The first native run confirmed the expected distinction between system-operation time and execution time. The example binds the full observed error text. A run that never reaches a refusal fails. +- Successful call counts, the first refused requested count, wall time and top-level completion time are recorded under the stated load. The failed request cannot return its exact completed-call count. +- A twenty-call control also inserts and reads one document per component call, asserts one read and one write per call, and records transaction metrics and completion usage. The example verifies F19 only; it does not compare against a plain helper for F4. diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md new file mode 100644 index 0000000..f3737b6 --- /dev/null +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md @@ -0,0 +1,29 @@ +--- +id: spec:facts.f19-nested-calls-share-time-budgets.probe-9-nested-time +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f19-nested-calls-share-time-budgets + verifies: spec:facts.f19-nested-calls-share-time-budgets +--- +# Nested calls reach a time boundary + +Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. + +## Intent + +- outcome: Establish the observed behavior on the pinned local backend. + +```gwt +Given a parent mutation that calls an empty nested mutation repeatedly +When the client increases the call count until the backend refuses the mutation +Then the error names {budget: "Your request timed out performing too many system operations."} time +And a direct computation fails with {cpu: "Function execution timed out (maximum duration: 1s)"} time +``` + +## Verification — executable + +- Every test owns a disposable native backend. Timings and call counts are recorded, not asserted, and say nothing about a hosted deployment. +- The first native run confirmed the expected distinction between system-operation time and execution time. The example binds the full observed error text. A run that never reaches a refusal fails. +- Successful call counts, the first refused requested count, wall time and top-level completion time are recorded under the stated load. The failed request cannot return its exact completed-call count. diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md new file mode 100644 index 0000000..356856a --- /dev/null +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md @@ -0,0 +1,34 @@ +--- +id: spec:facts.f19-nested-calls-share-time-budgets +kind: constraint +altitude: story +readiness: defined +relations: + refines: spec:facts.fact-ledger +--- +# Nested calls share time budgets + +F19 · Native backend tier · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. + +## Intent + +- outcome: Record the package declaration and native backend behavior with the limits of the evidence. (F19) + +### Open questions + +- [non-blocking] The public package declares no duration for the system-operation budget. The local timeout near fifteen seconds is an observation under load, not a hosted limit. The examples exercise mutations; query-only call trees and the allocation of JavaScript execution time among busy child functions remain unmeasured. (F19) + +## Constraints + +- statement: On the pinned native backend, repeated empty nested mutation calls and component mutation calls in one parent mutation can exceed one second of top-level elapsed execution and end with a system-operation timeout, distinct from the one-second execution timeout of direct computation. (F19) +- flavor: convex-fact +- target: evidence.status:probed +- measurableBy: Package source: `convex` 1.46.0, `node_modules/convex/src/server/impl/registration_impl.ts:63-68,755-770` sends nested and component mutation calls through the asynchronous `1.0/runUdf` syscall; `node_modules/convex/dist/esm-types/server/meta.d.ts:19-27` exposes byte and document metrics but no time metric; native backend tier: Probe 9 and Probe 11 on the pinned release, whose errors name the system-operation budget and the one-second execution budget. (F19) + +## Design + +- systemBoundary: A mutation that repeatedly calls an empty nested or component mutation fails with `Your request timed out performing too many system operations.` while its completion record reports zero documents and bytes read and written. A direct computation fails with `Function execution timed out (maximum duration: 1s)`. Both errors are bound, while elapsed times and requested call counts are recorded. (F19, Probe 9, Probe 11) +- nativeMeasurements: Under load, an initial local run completed 1,000 nested calls in 3.751 seconds and 1,000 component calls in 7.083 seconds; requests for 10,000 calls failed at 15.703 and 15.169 seconds respectively. These are top-level completion times on one local backend, not limits asserted by the examples. (F19, Probe 9, Probe 11) +- componentCost: Probe 11 compares transaction metrics before and after twenty component calls and records the top-level completion usage and time. Empty calls read and write zero documents. Each control call inserts one document and reads it back, costing one document read and one written; the initial run recorded 13.133 ms per empty call and 15.521 ms per control call under load. (F19, Probe 11) +- countBoundary: The probe increases the requested count until the backend refuses it, recording every successful count and the first failed request. The timeout can end execution before JavaScript catches the error, so the failed request's exact completed-call count is not observable by the return value. No count is bound as a platform limit. (F19, Probe 11) +- factCoverage: Probe 11 verifies this time-budget fact. It records component-call costs that inform the concern of F4, but does not verify F4's comparison against a plain helper because it has no helper baseline. (F19, F4, Probe 11) diff --git a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md new file mode 100644 index 0000000..014c195 --- /dev/null +++ b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md @@ -0,0 +1,29 @@ +--- +id: spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-oversized-row +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f20-pagination-checks-bytes-after-reading + verifies: spec:facts.f20-pagination-checks-bytes-after-reading +--- +# A page keeps the row that crosses its byte bound + +Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. + +## Intent + +- outcome: Establish the observed behavior on the pinned local backend. + +```gwt +Given a component list with two rows larger than the requested byte bound +When the parent reads the component paginator with byte bounds below and at one row +Then the first page keeps {rows: 1} row and reports {status: "SplitRequired"} +And the recorded bytes exceed the smaller requested bound by {overshoot: true} +``` + +## Verification — executable + +- Every test owns a disposable native backend. Timings and call counts are recorded, not asserted, and say nothing about a hosted deployment. +- The bound values are the expectation written before the first native run. A run that never reaches its boundary fails. +- The test also supplies a negative byte bound. The helper still reads and keeps one row, so the overshoot guarantee is stated only for finite non-negative bounds. diff --git a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md new file mode 100644 index 0000000..446ccb3 --- /dev/null +++ b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md @@ -0,0 +1,29 @@ +--- +id: spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-stream-budget +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f20-pagination-checks-bytes-after-reading + verifies: spec:facts.f20-pagination-checks-bytes-after-reading +--- +# A page includes stored system fields in its byte check + +Probe 10 · Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. + +## Intent + +- outcome: Read rows at their declared application-field budget through the context library's page options and record the system-field bytes beyond the page bound. + +```gwt +Given {count: 32} component rows whose application fields each occupy {budgetBytes: 262144} bytes +When the component paginator reads to the end cursor using the context library's page options +Then the page keeps {kept: 32} rows and marks {status: "SplitRequired"} +And its measured bytes are above the library byte bound by {exceeds: true} +``` + +## Verification — executable + +- The test uses `boundedPage`, `limitListPage` and `limitListBytes` from `src/context/queries.ts`, with an end cursor that removes the item count as a stopping condition. +- The bound values are the expectation written before the first native run. Stored sizes and measured bytes are recorded, not bound as constants. +- The table has the same application-field budget as a default single-mapping stream. Its stored rows also carry `_id` and `_creationTime`, which the helper includes in the byte count. diff --git a/design/specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md b/design/specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md new file mode 100644 index 0000000..7102247 --- /dev/null +++ b/design/specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md @@ -0,0 +1,32 @@ +--- +id: spec:facts.f20-pagination-checks-bytes-after-reading +kind: constraint +altitude: story +readiness: defined +relations: + refines: spec:facts.fact-ledger +--- +# Pagination checks bytes after reading + +F20 · Native backend tier · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. + +## Intent + +- outcome: Record the package declaration and native backend behavior with the limits of the evidence. (F20) + +### Open questions + +- [non-blocking] The bound applies to a plain index stream. Filtered, joined or mapped streams may charge reads whose rows are not retained, and the examples do not establish an overshoot bound for those compositions. A hosted deployment is not measured. (F20) + +## Constraints + +- statement: For a plain component index stream with a finite non-negative byte bound, convex-helpers keeps the next row and adds its stored document size before comparing the total with maximumBytesRead using greater-than-or-equal; a page can therefore keep and read beyond that bound by up to the last row's size and then report SplitRequired. (F20) +- flavor: convex-fact +- target: evidence.status:probed +- measurableBy: Package source: `convex-helpers` 0.1.124, `node_modules/convex-helpers/server/stream.js:292-315` for the late check and `:513-525` for `getDocumentSize`; native backend tier: Probe 10 on the pinned release; `src/context/queries.ts:49,100` passes `maximumBytesRead` through `boundedPage` and `spec:context.queries` is the Spec that rests on this fact. (F20) + +## Design + +- nativeOvershoot: With two stored rows of 8,281 bytes each, native queries with byte bounds of zero, one, 8,280 and 8,281 each read and keep the first row and report `SplitRequired`; the bound of zero does not mean read nothing, and equality also requests a split. The size is recorded, not asserted as a release limit. (F20, Probe 10) +- storedFields: `getDocumentSize` includes `_id` and `_creationTime`. A page computed from application-field budgets can exceed the claimed whole-page byte size even if each row's application fields fits its budget; the stream-budget example uses the actual `boundedPage`, `limitListPage` and `limitListBytes` helpers to measure that case. (F20, Probe 10) +- scopeOfBound: The last row is retained, so this option is a stopping threshold rather than a hard maximum of bytes already read or returned. A transaction that begins close to its backend read ceiling can fail while fetching that row before the helper gets to check the threshold. (F20, Probe 10) diff --git a/design/specs/facts/fact-ledger.sdp.md b/design/specs/facts/fact-ledger.sdp.md index 41384a7..a09dfb2 100644 --- a/design/specs/facts/fact-ledger.sdp.md +++ b/design/specs/facts/fact-ledger.sdp.md @@ -43,6 +43,7 @@ The four statuses map to the constraint target `evidence.status:documented`, `ev - factsRechecked: F3, F4, F5, F6 on 2026-09-29; F16 on 2026-09-30 against S6, changed from assumed (Fact ledger, F16) - factsAssumed: F17, with Probe 6; the doc also lists F14 and F15, which the corpus records as probed (Fact ledger, Probes) - factsProbed: F14 by Probe 2 and F15 by Probe 5, on 2026-10-01 on backend release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0; Probe 1 on F5, Probe 3 on F4 and Probe 4 on F13 ran the same day, and their results are recorded on those facts, whose status stays documented or rechecked (Fact ledger, Probes) +- additionalFactsProbed: F18 by Probe 8, F19 by Probe 9 and Probe 11, and F20 by Probe 10 on native backend `precompiled-2026-09-28-5c7cb5b`, with `convex` 1.46.0 and `convex-helpers` 0.1.124; each states its package source and the limits of the local evidence. (Fact ledger, F18, F19, F20) - sourcesRead: S1 to S13 by v0.1's author on 2026-09-29, S10, S14 and S15 again during the review the same day, and S2, S6, S8, S9, S10, S11 and the migrations and application-errors pages by the lead on 2026-09-30 (Sources) ## Verification — reviewed diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index b83e22d..c7ef8ba 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -6,6 +6,9 @@ readiness: defined relations: refines: spec:facts.fact-ledger dependsOn: + - spec:facts.f18-commit-timestamps + - spec:facts.f19-nested-calls-share-time-budgets + - spec:facts.f20-pagination-checks-bytes-after-reading - spec:facts.f14-convex-error-survives-nested-and-component-boundary - spec:facts.f15-parent-query-over-component-query-stays-reactive - spec:facts.f16-scheduled-functions-table-shows-failed-runs @@ -63,6 +66,11 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - Probe 6 runs a backfill batch through the migrations component while a live command writes the same read-model row, and checks that neither overwrites newer data and that the batch resumes after interruption; it serves D9 (Probe 6, F1, F17) - Probe 7, before Layer 3, takes a backup with pending scheduled functions, Workpool jobs and a Workflow run in flight, restores it, and records what survives; it reads `_scheduled_functions` for a failed run and confirms the states and the retention window; it serves D13 and D19 (Probe 7, F12, F16) +- Probe 8 writes commit timestamp placeholders in the parent and two components, compares committed rows and the runtime upper bound, and checks readback, return and scheduling behavior. (Probe 8, F18) +- Probe 9 repeats nested calls until a time boundary fails and compares its error with direct computation. (Probe 9, F19) +- Probe 10 reads component pages below and at a row's byte size and records the bytes kept beyond the requested bound. (Probe 10, F20) +- Probe 11 increases component calls per mutation to the backend's refusal, records counts and time, and measures documents read and written. (Probe 11, F19) + ## Design Each probe is a fixture-app test on a disposable backend. The probe app is separate from the example app and ships no test-only function to production. diff --git a/design/specs/foundation.pack.sdp.md b/design/specs/foundation.pack.sdp.md index 0f0cf32..c49c868 100644 --- a/design/specs/foundation.pack.sdp.md +++ b/design/specs/foundation.pack.sdp.md @@ -80,6 +80,14 @@ specs: - spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap - spec:facts.f16-scheduled-functions-table-shows-failed-runs - spec:facts.f17-migrations-fits-generation-backfill + - spec:facts.f18-commit-timestamps.probe-8-parent-and-components + - spec:facts.f18-commit-timestamps + - spec:facts.f19-nested-calls-share-time-budgets.probe-11-component-calls + - spec:facts.f19-nested-calls-share-time-budgets.probe-9-nested-time + - spec:facts.f19-nested-calls-share-time-budgets + - spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-oversized-row + - spec:facts.f20-pagination-checks-bytes-after-reading + - spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-stream-budget modelRefs: - spec:platform.vocabulary --- diff --git a/fixture/convex/_generated/api.d.ts b/fixture/convex/_generated/api.d.ts index 694907b..892daf9 100644 --- a/fixture/convex/_generated/api.d.ts +++ b/fixture/convex/_generated/api.d.ts @@ -8,6 +8,8 @@ * @module */ +import type * as bytePage from "../bytePage.js"; +import type * as callBudget from "../callBudget.js"; import type * as depotCommands from "../depotCommands.js"; import type * as depotQueries from "../depotQueries.js"; import type * as depotRelay from "../depotRelay.js"; @@ -26,9 +28,11 @@ import type * as parentList from "../parentList.js"; import type * as readCost from "../readCost.js"; import type * as readModels from "../readModels.js"; import type * as rejectionCommands from "../rejectionCommands.js"; +import type * as scheduledRows from "../scheduledRows.js"; import type * as summaries from "../summaries.js"; import type * as summarizedTwice from "../summarizedTwice.js"; import type * as switches from "../switches.js"; +import type * as timestamps from "../timestamps.js"; import type * as usage from "../usage.js"; import type { @@ -38,6 +42,8 @@ import type { } from "convex/server"; declare const fullApi: ApiFromModules<{ + bytePage: typeof bytePage; + callBudget: typeof callBudget; depotCommands: typeof depotCommands; depotQueries: typeof depotQueries; depotRelay: typeof depotRelay; @@ -56,9 +62,11 @@ declare const fullApi: ApiFromModules<{ readCost: typeof readCost; readModels: typeof readModels; rejectionCommands: typeof rejectionCommands; + scheduledRows: typeof scheduledRows; summaries: typeof summaries; summarizedTwice: typeof summarizedTwice; switches: typeof switches; + timestamps: typeof timestamps; usage: typeof usage; }>; @@ -90,6 +98,7 @@ export declare const internal: FilterApi< export declare const components: { annex: import("../annex/_generated/component.js").ComponentApi<"annex">; + annexClock: import("../annex/_generated/component.js").ComponentApi<"annexClock">; depot: import("../depot/_generated/component.js").ComponentApi<"depot">; yard: import("../yard/_generated/component.js").ComponentApi<"yard">; }; diff --git a/fixture/convex/annex/_generated/api.ts b/fixture/convex/annex/_generated/api.ts index e9076bc..bbb7448 100644 --- a/fixture/convex/annex/_generated/api.ts +++ b/fixture/convex/annex/_generated/api.ts @@ -8,12 +8,16 @@ * @module */ +import type * as bytePage from "../bytePage.js"; +import type * as callBudget from "../callBudget.js"; import type * as failures from "../failures.js"; import type * as identity from "../identity.js"; import type * as limits from "../limits.js"; import type * as list from "../list.js"; import type * as notes from "../notes.js"; import type * as readCost from "../readCost.js"; +import type * as scheduledRows from "../scheduledRows.js"; +import type * as timestamps from "../timestamps.js"; import type { ApiFromModules, @@ -23,12 +27,16 @@ import type { import { anyApi, componentsGeneric } from "convex/server"; const fullApi: ApiFromModules<{ + bytePage: typeof bytePage; + callBudget: typeof callBudget; failures: typeof failures; identity: typeof identity; limits: typeof limits; list: typeof list; notes: typeof notes; readCost: typeof readCost; + scheduledRows: typeof scheduledRows; + timestamps: typeof timestamps; }> = anyApi as any; /** diff --git a/fixture/convex/annex/_generated/component.ts b/fixture/convex/annex/_generated/component.ts index 339741f..5dd225d 100644 --- a/fixture/convex/annex/_generated/component.ts +++ b/fixture/convex/annex/_generated/component.ts @@ -23,6 +23,45 @@ import type { FunctionReference } from "convex/server"; */ export type ComponentApi = { + bytePage: { + bounded: FunctionReference< + "query", + "internal", + { budgetBytes: number }, + any, + Name + >; + read: FunctionReference< + "query", + "internal", + { maximumBytesRead: number }, + any, + Name + >; + seed: FunctionReference< + "mutation", + "internal", + { budgetBytes: number; count: number; first: number }, + null, + Name + >; + }; + callBudget: { + empty: FunctionReference< + "mutation", + "internal", + { index: number }, + number, + Name + >; + write: FunctionReference< + "mutation", + "internal", + { index: number }, + number, + Name + >; + }; failures: { throwAfterWrite: FunctionReference< "mutation", @@ -120,4 +159,17 @@ export type ComponentApi = >; seed: FunctionReference<"mutation", "internal", {}, string, Name>; }; + scheduledRows: { + schedule: FunctionReference<"mutation", "internal", {}, string, Name>; + }; + timestamps: { + read: FunctionReference<"query", "internal", {}, any, Name>; + write: FunctionReference< + "mutation", + "internal", + { label: string }, + { numericError: string; unresolved: boolean }, + Name + >; + }; }; diff --git a/fixture/convex/annex/bytePage.ts b/fixture/convex/annex/bytePage.ts new file mode 100644 index 0000000..7137920 --- /dev/null +++ b/fixture/convex/annex/bytePage.ts @@ -0,0 +1,56 @@ +import { + boundedPage, + limitListPage, + limitListBytes, +} from "../../../src/context/queries.js"; +import { paginator } from "convex-helpers/server/pagination"; +import { getConvexSize, v } from "convex/values"; +import { query, mutation } from "./_generated/server.js"; +import schema from "./schema.js"; +export const read = query({ + args: { maximumBytesRead: v.number() }, + returns: v.any(), + handler: async (ctx, { maximumBytesRead }) => { + const before = await ctx.meta.getTransactionMetrics(); + const result = await paginator(ctx.db, schema) + .query("rows") + .withIndex("by_position") + .paginate({ cursor: null, numItems: 10, maximumBytesRead }); + return { result, before, after: await ctx.meta.getTransactionMetrics() }; + }, +}); + +export const bounded = query({ + args: { budgetBytes: v.number() }, + returns: v.any(), + handler: async (ctx, { budgetBytes }) => { + const before = await ctx.meta.getTransactionMetrics(); + const options = boundedPage( + { cursor: null, numItems: 100, endCursor: "[]" }, + { items: limitListPage(budgetBytes), bytes: limitListBytes }, + ); + const result = await paginator(ctx.db, schema) + .query("rows") + .withIndex("by_position") + .paginate(options); + return { + count: result.page.length, + status: result.pageStatus, + options, + before, + after: await ctx.meta.getTransactionMetrics(), + }; + }, +}); +export const seed = mutation({ + args: { first: v.number(), count: v.number(), budgetBytes: v.number() }, + returns: v.null(), + handler: async (ctx, { first, count, budgetBytes }) => { + for (let position = first; position < first + count; position++) { + const row = { position, label: "" }; + row.label = "x".repeat(budgetBytes - getConvexSize(row)); + await ctx.db.insert("rows", row); + } + return null; + }, +}); diff --git a/fixture/convex/annex/callBudget.ts b/fixture/convex/annex/callBudget.ts new file mode 100644 index 0000000..6e1fe7a --- /dev/null +++ b/fixture/convex/annex/callBudget.ts @@ -0,0 +1,16 @@ +import { v } from "convex/values"; +import { mutation } from "./_generated/server.js"; +export const empty = mutation({ + args: { index: v.number() }, + returns: v.number(), + handler: (_, { index }) => index, +}); +export const write = mutation({ + args: { index: v.number() }, + returns: v.number(), + handler: async (ctx, { index }) => { + const id = await ctx.db.insert("samples", { value: index }); + const row = await ctx.db.get("samples", id); + return row!.value; + }, +}); diff --git a/fixture/convex/annex/scheduledRows.ts b/fixture/convex/annex/scheduledRows.ts new file mode 100644 index 0000000..21e992a --- /dev/null +++ b/fixture/convex/annex/scheduledRows.ts @@ -0,0 +1,15 @@ +import type { Id } from "./_generated/dataModel.js"; +import { v } from "convex/values"; +import { internal } from "./_generated/api.js"; +import { internalMutation, mutation } from "./_generated/server.js"; +export const receive = internalMutation({ + args: {}, + returns: v.null(), + handler: () => null, +}); +export const schedule = mutation({ + args: {}, + returns: v.id("_scheduled_functions"), + handler: (ctx): Promise> => + ctx.scheduler.runAfter(3600000, internal.scheduledRows.receive, {}), +}); diff --git a/fixture/convex/annex/schema.ts b/fixture/convex/annex/schema.ts index 8b9eaa2..ecb434b 100644 --- a/fixture/convex/annex/schema.ts +++ b/fixture/convex/annex/schema.ts @@ -1,6 +1,10 @@ import { defineSchema, defineTable } from "convex/server"; import { v } from "convex/values"; export default defineSchema({ + timestamps: defineTable({ label: v.string(), commitTs: v.any() }).index( + "by_commit", + ["commitTs"], + ), notes: defineTable({ source: v.string() }), throwerWrites: defineTable({ by: v.string() }).index("by_by", ["by"]), samples: defineTable({ value: v.number() }), diff --git a/fixture/convex/annex/timestamps.ts b/fixture/convex/annex/timestamps.ts new file mode 100644 index 0000000..b0bddb6 --- /dev/null +++ b/fixture/convex/annex/timestamps.ts @@ -0,0 +1,31 @@ +import { v } from "convex/values"; +import { mutation, query } from "./_generated/server.js"; +export const write = mutation({ + args: { label: v.string() }, + returns: v.object({ unresolved: v.boolean(), numericError: v.string() }), + handler: async (ctx, { label }) => { + const id = await ctx.db.insert("timestamps", { + label, + commitTs: ctx.db.vars.commitTs, + }); + const row = await ctx.db.get("timestamps", id); + let numericError = ""; + try { + Number(row?.commitTs); + } catch (error) { + numericError = String(error); + } + return { unresolved: row?.commitTs === ctx.db.vars.commitTs, numericError }; + }, +}); +export const read = query({ + args: {}, + returns: v.any(), + handler: async (ctx) => ({ + rows: await ctx.db.query("timestamps").withIndex("by_commit").collect(), + // The runtime implements this internal method; the public declarations omit it. + upper: ( + ctx.meta as typeof ctx.meta & { getSnapshotTs(): bigint } + ).getSnapshotTs(), + }), +}); diff --git a/fixture/convex/bytePage.ts b/fixture/convex/bytePage.ts new file mode 100644 index 0000000..dc091bc --- /dev/null +++ b/fixture/convex/bytePage.ts @@ -0,0 +1,9 @@ +import { v } from "convex/values"; +import { components } from "./_generated/api.js"; +import { query } from "./_generated/server.js"; +export const read = query({ + args: { maximumBytesRead: v.number() }, + returns: v.any(), + handler: (ctx, args): Promise => + ctx.runQuery(components.annex.bytePage.read, args), +}); diff --git a/fixture/convex/callBudget.ts b/fixture/convex/callBudget.ts new file mode 100644 index 0000000..560dd13 --- /dev/null +++ b/fixture/convex/callBudget.ts @@ -0,0 +1,54 @@ +import type { TransactionMetrics } from "convex/server"; +import { v } from "convex/values"; +import { components, internal } from "./_generated/api.js"; +import { internalMutation, mutation } from "./_generated/server.js"; +export const empty = internalMutation({ + args: { index: v.number() }, + returns: v.number(), + handler: (_, { index }) => index, +}); +export const repeat = mutation({ + args: { + path: v.union(v.literal("nested"), v.literal("component")), + count: v.number(), + write: v.boolean(), + }, + returns: v.any(), + handler: async ( + ctx, + { path, count, write }, + ): Promise<{ + completed: number; + before: TransactionMetrics; + after: TransactionMetrics; + }> => { + const before = await ctx.meta.getTransactionMetrics(); + let completed = 0; + try { + for (let index = 0; index < count; index++) { + const answer: number = await ctx.runMutation( + path === "nested" + ? internal.callBudget.empty + : write + ? components.annex.callBudget.write + : components.annex.callBudget.empty, + { index }, + ); + if (answer !== index) throw new Error("Call returned another index"); + completed++; + } + } catch (error) { + throw new Error(`Completed ${completed} calls: ${String(error)}`); + } + return { completed, before, after: await ctx.meta.getTransactionMetrics() }; + }, +}); +export const compute = mutation({ + args: {}, + returns: v.number(), + handler: () => { + let sum = 0; + for (let index = 0; index < 1e12; index++) sum += Math.sqrt(index); + return sum; + }, +}); diff --git a/fixture/convex/convex.config.ts b/fixture/convex/convex.config.ts index 69121e2..d247219 100644 --- a/fixture/convex/convex.config.ts +++ b/fixture/convex/convex.config.ts @@ -4,6 +4,7 @@ import depot from "./depot/convex.config.js"; import yard from "./yard/convex.config.js"; const app = defineApp(); app.use(annex); +app.use(annex, { name: "annexClock" }); // The fixture contexts. Each mount name is its contextId. The yard is there so a use case can call // two contexts. app.use(depot, { name: "depot" }); diff --git a/fixture/convex/scheduledRows.ts b/fixture/convex/scheduledRows.ts new file mode 100644 index 0000000..6175f5b --- /dev/null +++ b/fixture/convex/scheduledRows.ts @@ -0,0 +1,23 @@ +import { v } from "convex/values"; +import { internal, components } from "./_generated/api.js"; +import { internalMutation, mutation } from "./_generated/server.js"; +export const receive = internalMutation({ + args: {}, + returns: v.null(), + handler: () => null, +}); +export const schedule = mutation({ + args: {}, + returns: v.any(), + handler: async (ctx): Promise => ({ + parent: await ctx.scheduler.runAfter( + 3600000, + internal.scheduledRows.receive, + {}, + ), + component: await ctx.runMutation( + components.annex.scheduledRows.schedule, + {}, + ), + }), +}); diff --git a/fixture/convex/schema.ts b/fixture/convex/schema.ts index eb5643c..5f4aeeb 100644 --- a/fixture/convex/schema.ts +++ b/fixture/convex/schema.ts @@ -5,6 +5,10 @@ import { readModelTables, rowConventions } from "../../src/read-model/index.js"; import { documentTitleFields } from "./documentTitles.js"; import { documentSummaryFields } from "./summaries.js"; export default defineSchema({ + timestamps: defineTable({ label: v.string(), commitTs: v.any() }).index( + "by_commit", + ["commitTs"], + ), notes: defineTable({ source: v.string() }), depthRows: defineTable({ trial: v.string() }).index("by_trial", ["trial"]), markers: defineTable({ trial: v.string() }).index("by_trial", ["trial"]), diff --git a/fixture/convex/timestamps.ts b/fixture/convex/timestamps.ts new file mode 100644 index 0000000..289c576 --- /dev/null +++ b/fixture/convex/timestamps.ts @@ -0,0 +1,69 @@ +import { v } from "convex/values"; +import { components, internal } from "./_generated/api.js"; +import { mutation, query, internalMutation } from "./_generated/server.js"; +export const write = mutation({ + args: { label: v.string() }, + returns: v.any(), + handler: async (ctx, { label }): Promise => { + const first = await ctx.runMutation(components.annex.timestamps.write, { + label, + }); + const id = await ctx.db.insert("timestamps", { + label, + commitTs: ctx.db.vars.commitTs, + }); + const second = await ctx.runMutation( + components.annexClock.timestamps.write, + { label }, + ); + const row = await ctx.db.get("timestamps", id); + return { + first, + second, + unresolved: row?.commitTs === ctx.db.vars.commitTs, + }; + }, +}); +export const read = query({ + args: {}, + returns: v.any(), + handler: async (ctx): Promise => ({ + rows: await ctx.db.query("timestamps").withIndex("by_commit").collect(), + first: await ctx.runQuery(components.annex.timestamps.read, {}), + second: await ctx.runQuery(components.annexClock.timestamps.read, {}), + // The runtime implements this internal method; the public declarations omit it. + upper: ( + ctx.meta as typeof ctx.meta & { getSnapshotTs(): bigint } + ).getSnapshotTs(), + }), +}); +export const receive = internalMutation({ + args: { value: v.any() }, + returns: v.null(), + handler: () => null, +}); +export const returnPlaceholder = mutation({ + args: {}, + returns: v.any(), + handler: async (ctx) => { + await ctx.db.insert("timestamps", { + label: "returned", + commitTs: ctx.db.vars.commitTs, + }); + return ctx.db.vars.commitTs; + }, +}); +export const schedulePlaceholder = mutation({ + args: {}, + returns: v.null(), + handler: async (ctx) => { + await ctx.db.insert("timestamps", { + label: "scheduled", + commitTs: ctx.db.vars.commitTs, + }); + await ctx.scheduler.runAfter(60000, internal.timestamps.receive, { + value: ctx.db.vars.commitTs, + }); + return null; + }, +}); diff --git a/tests/native/call-budget.ts b/tests/native/call-budget.ts new file mode 100644 index 0000000..36bab23 --- /dev/null +++ b/tests/native/call-budget.ts @@ -0,0 +1,133 @@ +import type { TransactionMetrics } from "convex/server"; +import { expect } from "vitest"; +import { api } from "../../fixture/convex/_generated/api.js"; +import type { Backend } from "../../harness/backend.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend, measure, required } from "../../harness/native.js"; +export interface BudgetWorld { + backend?: Backend; + path?: "nested" | "component"; + error?: string; + cpuError?: string; +} +export async function prepareBudget( + world: BudgetWorld, + path: "nested" | "component", +) { + world.backend = await fixtureBackend(); + world.path = path; +} +export async function reachBudget(world: BudgetWorld) { + const backend = required(world.backend, "backend"); + const client = ordinaryClient(backend.url); + const path = required(world.path, "path"); + for (const count of [100, 1000, 2000, 4000, 8000, 16000, 32000]) { + const mark = await backend.admin.logMark(); + const start = performance.now(); + let result: + | { + completed: number; + before: TransactionMetrics; + after: TransactionMetrics; + } + | undefined; + try { + result = await client.mutation(api.callBudget.repeat, { + path, + count, + write: false, + }); + } catch (error) { + world.error = String(error); + } + const wallMs = performance.now() - start; + const records = await backend.admin.completionsSince(mark, (entries) => + entries.some((entry) => entry.identifier === "callBudget:repeat"), + ); + const own = required( + records.find((entry) => entry.identifier === "callBudget:repeat"), + "completion", + ); + measure(`${path}-${count}`, { + count, + completed: result?.completed ?? null, + wallMs, + executionSeconds: own.executionTime, + usageStats: own.usageStats, + error: world.error ?? null, + underLoad: process.env["LIBAR_TIMINGS_UNDER_LOAD"] !== "0", + }); + if (world.error !== undefined) break; + expect(result?.completed).toBe(count); + expect(result?.after.documentsRead.used).toBe(0); + expect(result?.after.documentsWritten.used).toBe(0); + } + expect(world.error, "The run never reached a time boundary").toBeDefined(); +} +export async function computeBudget(world: BudgetWorld, expected: string) { + const backend = required(world.backend, "backend"); + const client = ordinaryClient(backend.url); + const start = performance.now(); + try { + await client.mutation(api.callBudget.compute, {}); + } catch (error) { + world.cpuError = String(error); + } + measure("computationBoundary", { + error: world.cpuError ?? null, + wallMs: performance.now() - start, + underLoad: process.env["LIBAR_TIMINGS_UNDER_LOAD"] !== "0", + }); + expect( + world.cpuError, + "The computation did not reach its boundary", + ).toBeDefined(); + expect(world.cpuError).toContain(expected); +} +export async function measureCalls( + world: BudgetWorld, + reads: number, + writes: number, +) { + const backend = required(world.backend, "backend"); + const client = ordinaryClient(backend.url); + for (const write of [false, true]) { + const count = 20; + const mark = await backend.admin.logMark(); + const start = performance.now(); + const result: { + completed: number; + before: TransactionMetrics; + after: TransactionMetrics; + } = await client.mutation(api.callBudget.repeat, { + path: "component", + count, + write, + }); + const wallMs = performance.now() - start; + const records = await backend.admin.completionsSince(mark, (entries) => + entries.some((entry) => entry.identifier === "callBudget:repeat"), + ); + const own = required( + records.find((entry) => entry.identifier === "callBudget:repeat"), + "completion", + ); + const documentsRead = + result.after.documentsRead.used - result.before.documentsRead.used; + const documentsWritten = + result.after.documentsWritten.used - result.before.documentsWritten.used; + measure(`componentCost-${write ? "write" : "empty"}`, { + count, + documentsRead, + documentsWritten, + usageStats: own.usageStats, + executionSeconds: own.executionTime, + perCallMs: (own.executionTime * 1000) / count, + wallMs, + underLoad: process.env["LIBAR_TIMINGS_UNDER_LOAD"] !== "0", + }); + expect(result.completed).toBe(count); + expect(documentsRead).toBe(write ? count : count * reads); + expect(documentsWritten).toBe(write ? count : count * writes); + } +} diff --git a/tests/native/f18-commit-timestamps.probe-8-parent-and-components.test.ts b/tests/native/f18-commit-timestamps.probe-8-parent-and-components.test.ts new file mode 100644 index 0000000..167c616 --- /dev/null +++ b/tests/native/f18-commit-timestamps.probe-8-parent-and-components.test.ts @@ -0,0 +1,140 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import type { QueryMeta } from "convex/server"; +import { expect } from "vitest"; +import { probe8ParentAndComponentsContract as contract } from "../../generated/contracts/facts.f18-commit-timestamps.probe-8-parent-and-components.contract.js"; +import { api } from "../../fixture/convex/_generated/api.js"; +import type { Backend } from "../../harness/backend.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend, measure, required } from "../../harness/native.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f18-commit-timestamps.probe-8-parent-and-components", + ), + verifies: ref( + "spec:facts.f18-commit-timestamps.probe-8-parent-and-components", + ), +}); +void anchor; +type Rows = { rows: { label: string; commitTs: bigint }[]; upper: bigint }; +interface World { + backend?: Backend; + read?: Rows & { first: Rows; second: Rows }; + errors?: string[]; + returned?: unknown; +} +// Compiled evidence: the published QueryMeta declaration has no upper-bound method. +const declared: "getSnapshotTs" extends keyof QueryMeta ? true : false = false; +bindExample(contract, (): World => ({}), { + "a parent and two components that write commit timestamp placeholders": + async (world) => { + world.backend = await fixtureBackend(); + expect(declared).toBe(false); + }, + "three mutations commit in sequence and a query reads their indexed rows": + async (world) => { + const client = ordinaryClient(required(world.backend, "backend").url); + for (const label of ["one", "two", "three"]) { + const result = await client.mutation(api.timestamps.write, { label }); + expect(result).toMatchObject({ + unresolved: true, + first: { unresolved: true }, + second: { unresolved: true }, + }); + expect(result).toMatchObject({ + first: { numericError: expect.stringContaining("unresolved") }, + second: { numericError: expect.stringContaining("unresolved") }, + }); + measure(`readback-${label}`, result as Parameters[1]); + } + world.read = (await client.query(api.timestamps.read, {})) as NonNullable< + World["read"] + >; + measure( + "committedTimestamps", + JSON.parse( + JSON.stringify(world.read, (_, value: unknown) => + typeof value === "bigint" ? value.toString() : value, + ), + ) as Parameters[1], + ); + world.errors = []; + for (const reference of [ + api.timestamps.returnPlaceholder, + api.timestamps.schedulePlaceholder, + ]) { + try { + const value = await client.mutation(reference, {}); + world.returned = value; + measure("returnedPlaceholder", { + type: typeof value, + text: String(value), + }); + world.errors.push(""); + } catch (error) { + world.errors.push(String(error)); + } + } + measure("placeholderErrors", world.errors); + }, + "each transaction has {shared} timestamp across the three tables and later transactions have larger timestamps": + (world, { shared }) => { + const read = required(world.read, "query"); + expect(read.rows).toHaveLength(3); + expect(read.first.rows).toEqual( + expect.arrayContaining( + read.rows.map(({ label, commitTs }) => + expect.objectContaining({ label, commitTs }), + ), + ), + ); + expect(read.second.rows).toEqual( + expect.arrayContaining( + read.rows.map(({ label, commitTs }) => + expect.objectContaining({ label, commitTs }), + ), + ), + ); + for (const [index, row] of read.rows.entries()) { + expect(typeof row.commitTs).toBe("bigint"); + expect(row.commitTs <= read.upper).toBe(true); + if (index > 0) + expect(row.commitTs > read.rows[index - 1]!.commitTs).toBe(true); + expect( + row.commitTs === read.first.rows[index]!.commitTs && + row.commitTs === read.second.rows[index]!.commitTs, + ).toBe(shared); + } + expect(read.first.upper).toBe(read.upper); + expect(read.second.upper).toBe(read.upper); + }, + "returning a placeholder {returned} and scheduling it is {refused}": async ( + world, + { returned, refused }, + ) => { + expect( + typeof world.returned === "bigint" + ? "resolves to a bigint" + : String(world.returned), + ).toBe(returned); + expect(required(world.errors, "errors")[1]?.includes("$commitTs")).toBe( + refused, + ); + const backend = required(world.backend, "backend"); + const rows = await backend.admin.readTable("timestamps"); + expect(rows).toHaveLength(4); + expect(rows.find((row) => row.label === "returned")?.commitTs).toBe( + world.returned, + ); + expect( + (world.returned as bigint) > required(world.read, "query").upper, + ).toBe(true); + expect(await backend.admin.readTable("_scheduled_functions")).toHaveLength( + 0, + ); + }, +}); diff --git a/tests/native/f19-nested-calls-share-time-budgets.probe-11-component-calls.test.ts b/tests/native/f19-nested-calls-share-time-budgets.probe-11-component-calls.test.ts new file mode 100644 index 0000000..09e7ce6 --- /dev/null +++ b/tests/native/f19-nested-calls-share-time-budgets.probe-11-component-calls.test.ts @@ -0,0 +1,35 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { probe11ComponentCallsContract as contract } from "../../generated/contracts/facts.f19-nested-calls-share-time-budgets.probe-11-component-calls.contract.js"; +import { + prepareBudget, + reachBudget, + measureCalls, + type BudgetWorld, +} from "./call-budget.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f19-nested-calls-share-time-budgets.probe-11-component-calls", + ), + verifies: ref( + "spec:facts.f19-nested-calls-share-time-budgets.probe-11-component-calls", + ), +}); +void anchor; +bindExample(contract, (): BudgetWorld => ({}), { + "a parent mutation that calls an empty component mutation repeatedly": async ( + world, + ) => prepareBudget(world, "component"), + "the client increases the call count until the backend refuses the mutation": + reachBudget, + "the error names {budget} time": (world, { budget }) => { + expect(world.error).toContain(budget); + }, + "each empty component call reads {reads} and writes {writes} documents": + async (world, { reads, writes }) => measureCalls(world, reads, writes), +}); diff --git a/tests/native/f19-nested-calls-share-time-budgets.probe-9-nested-time.test.ts b/tests/native/f19-nested-calls-share-time-budgets.probe-9-nested-time.test.ts new file mode 100644 index 0000000..d5d03f8 --- /dev/null +++ b/tests/native/f19-nested-calls-share-time-budgets.probe-9-nested-time.test.ts @@ -0,0 +1,35 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { probe9NestedTimeContract as contract } from "../../generated/contracts/facts.f19-nested-calls-share-time-budgets.probe-9-nested-time.contract.js"; +import { + prepareBudget, + reachBudget, + computeBudget, + type BudgetWorld, +} from "./call-budget.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f19-nested-calls-share-time-budgets.probe-9-nested-time", + ), + verifies: ref( + "spec:facts.f19-nested-calls-share-time-budgets.probe-9-nested-time", + ), +}); +void anchor; +bindExample(contract, (): BudgetWorld => ({}), { + "a parent mutation that calls an empty nested mutation repeatedly": async ( + world, + ) => prepareBudget(world, "nested"), + "the client increases the call count until the backend refuses the mutation": + reachBudget, + "the error names {budget} time": (world, { budget }) => { + expect(world.error).toContain(budget); + }, + "a direct computation fails with {cpu} time": async (world, { cpu }) => + computeBudget(world, cpu), +}); diff --git a/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.test.ts b/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.test.ts new file mode 100644 index 0000000..2cb596d --- /dev/null +++ b/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.test.ts @@ -0,0 +1,92 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import type { TransactionMetrics } from "convex/server"; +import { getDocumentSize } from "convex/values"; +import { expect } from "vitest"; +import { probe10OversizedRowContract as contract } from "../../generated/contracts/facts.f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.contract.js"; +import { api } from "../../fixture/convex/_generated/api.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import type { Backend } from "../../harness/backend.js"; +import { fixtureBackend, measure, required } from "../../harness/native.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f20-pagination-checks-bytes-after-reading.probe-10-oversized-row", + ), + verifies: ref( + "spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-oversized-row", + ), +}); +void anchor; +type Result = { + result: { page: Record[]; pageStatus: string }; + before: TransactionMetrics; + after: TransactionMetrics; +}; +interface World { + backend?: Backend; + pages?: { bound: number; read: Result }[]; +} +bindExample(contract, (): World => ({}), { + "a component list with two rows larger than the requested byte bound": async ( + world, + ) => { + world.backend = await fixtureBackend(); + await world.backend.admin.run( + "list:insert", + { + rows: [ + { position: 1, label: "a".repeat(8192) }, + { position: 2, label: "b".repeat(8192) }, + ], + }, + { component: "annex" }, + ); + }, + "the parent reads the component paginator with byte bounds below and at one row": + async (world) => { + const backend = required(world.backend, "backend"); + const stored = await backend.admin.readTable("rows", { + component: "annex", + }); + const size = getDocumentSize(stored[0]!); + world.pages = []; + for (const bound of [-1, 0, 1, size - 1, size]) { + const read = (await ordinaryClient(backend.url).query( + api.bytePage.read, + { maximumBytesRead: bound }, + )) as unknown as Result; + world.pages.push({ bound, read }); + measure(`page-${bound}`, { + bound, + storedSize: size, + kept: read.result.page.length, + status: read.result.pageStatus, + bytesRead: read.after.bytesRead.used - read.before.bytesRead.used, + documentsRead: + read.after.documentsRead.used - read.before.documentsRead.used, + }); + } + }, + "the first page keeps {rows} row and reports {status}": ( + world, + { rows, status }, + ) => { + for (const { read } of required(world.pages, "pages")) { + expect(read.result.page).toHaveLength(rows); + expect(read.result.pageStatus).toBe(status); + } + }, + "the recorded bytes exceed the smaller requested bound by {overshoot}": ( + world, + { overshoot }, + ) => { + for (const { bound, read } of required(world.pages, "pages").slice(0, -1)) + expect( + read.after.bytesRead.used - read.before.bytesRead.used > bound, + ).toBe(overshoot); + }, +}); diff --git a/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.test.ts b/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.test.ts new file mode 100644 index 0000000..8ff04d4 --- /dev/null +++ b/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.test.ts @@ -0,0 +1,74 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import type { TransactionMetrics } from "convex/server"; +import { expect } from "vitest"; +import { probe10StreamBudgetContract as contract } from "../../generated/contracts/facts.f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.contract.js"; +import type { Backend } from "../../harness/backend.js"; +import { fixtureBackend, measure, required } from "../../harness/native.js"; +import { paceAfterWrite } from "../../harness/wait.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f20-pagination-checks-bytes-after-reading.probe-10-stream-budget", + ), + verifies: ref( + "spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-stream-budget", + ), +}); +void anchor; +interface World { + backend?: Backend; + budgetBytes?: number; + result?: { + count: number; + status: string; + options: { maximumBytesRead: number }; + before: TransactionMetrics; + after: TransactionMetrics; + }; +} +bindExample(contract, (): World => ({}), { + "{count} component rows whose application fields each occupy {budgetBytes} bytes": + async (world, { count, budgetBytes }) => { + world.backend = await fixtureBackend(); + world.budgetBytes = budgetBytes; + for (let first = 0; first < count; first += 4) { + const size = Math.min(4, count - first); + await world.backend.admin.run( + "bytePage:seed", + { first, count: size, budgetBytes }, + { component: "annex" }, + ); + await paceAfterWrite(size * (budgetBytes + 256)); + } + }, + "the component paginator reads to the end cursor using the context library's page options": + async (world) => { + const result = await required(world.backend, "backend").admin.run( + "bytePage:bounded", + { budgetBytes: required(world.budgetBytes, "budget") }, + { component: "annex" }, + ); + world.result = result as unknown as NonNullable; + measure("boundedPage", result as Parameters[1]); + }, + "the page keeps {kept} rows and marks {status}": ( + world, + { kept, status }, + ) => { + expect(world.result).toMatchObject({ count: kept, status }); + }, + "its measured bytes are above the library byte bound by {exceeds}": ( + world, + { exceeds }, + ) => { + const result = required(world.result, "result"); + expect( + result.after.bytesRead.used - result.before.bytesRead.used > + result.options.maximumBytesRead, + ).toBe(exceeds); + }, +}); diff --git a/tests/native/first-experiment-steps.ts b/tests/native/first-experiment-steps.ts index ada32a7..b43420a 100644 --- a/tests/native/first-experiment-steps.ts +++ b/tests/native/first-experiment-steps.ts @@ -21,6 +21,7 @@ import type { CompletionRecord } from "../../harness/admin.js"; import type { Backend } from "../../harness/backend.js"; import { ordinaryClient } from "../../harness/clients.js"; import { measure, productionBackend, required } from "../../harness/native.js"; +import { scheduledRows } from "./scheduled-rows.js"; import type { MutationCtx } from "../../src/command/index.js"; export const tenantId = "t-1"; export const subject = "user-1"; @@ -76,6 +77,10 @@ export interface Placed { window: CompletionRecord[]; summaryKeys: string[]; } +const scheduledByOrder = new WeakMap< + Placed, + Awaited> +>(); export interface ExperimentWorld { backend?: Backend; client?: ConvexHttpClient; @@ -199,7 +204,7 @@ export async function place( ), "the command's completion record", ); - return { + const placed: Placed = { orderId, lines, response, @@ -208,6 +213,8 @@ export async function place( window, summaryKeys: summaries.page.map((row) => row.key), }; + scheduledByOrder.set(placed, await scheduledRows(backend)); + return placed; } export async function placeOrderRuns( world: ExperimentWorld, @@ -244,15 +251,13 @@ export function commitsAre(world: ExperimentWorld, commits: number) { expect(placed.request).toEqual(committed); } } -// Nothing ran between the command and the read sent after it returned, and that read already shows -// the command's summary row: the command wrote it, and no job did. +// Admin reads of all scheduled-function tables include delayed jobs. The ordinary query already +// shows the command's summary row. export function projectionJobsAre(world: ExperimentWorld, jobs: number) { for (const placed of sent(world)) { - const others = placed.window.filter( - (record) => - record !== placed.own && record.identifier !== summariesIdentifier, - ); - expect(others).toHaveLength(jobs); + expect( + required(scheduledByOrder.get(placed), "the scheduled-function rows"), + ).toHaveLength(jobs); expect(placed.summaryKeys).toContain(placed.orderId); } } diff --git a/tests/native/read-models.committed-state-read-through-query.test.ts b/tests/native/read-models.committed-state-read-through-query.test.ts index 14ab689..a1f3961 100644 --- a/tests/native/read-models.committed-state-read-through-query.test.ts +++ b/tests/native/read-models.committed-state-read-through-query.test.ts @@ -10,6 +10,7 @@ import { committedStateReadThroughQueryContract as contract } from "../../genera import { api } from "../../example/convex/_generated/api.js"; import type { LogMark } from "../../harness/admin.js"; import { required } from "../../harness/native.js"; +import { scheduledRows } from "./scheduled-rows.js"; import { line, orderWorld, @@ -28,7 +29,7 @@ const anchor = specTest({ void anchor; // PlaceOrder commits in its own mutation, and the parent query getOrder, which relays the Orders // context's get, answers the committed order as soon as the command has returned. The function log -// from the command to that read shows that nothing but the two calls the client made ran. +// identifies both client calls. Admin reads of the scheduled-function tables check that no job exists. type Response = FunctionReturnType; interface World { order?: OrderWorld; @@ -37,9 +38,6 @@ interface World { } const orderId = "order-1"; const lines = [line("sku-1", 2, 250), line("sku-2", 1, 40)]; -// A record a client caused: a call over HTTP, or a query or mutation over the socket. A scheduled -// function, a cron or an action is a worker. -const clientCallers = new Set(["HttpApi", "SyncWorker"]); const placeOrderIdentifier = "ordering:placeOrder"; const getOrderIdentifier = "orderQueries:getOrder"; bindExample(contract, (): World => ({}), { @@ -97,8 +95,7 @@ bindExample(contract, (): World => ({}), { version: orderVersions[0], }); }, - // Every record in the function log from the command to the read: the command's own mutation and - // the read's query, both caused by the client. + // The log identifies the two client calls; the tables include delayed scheduled functions. "the number of workers, jobs or queues that ran is {workers}": async ( world, { workers }, @@ -110,14 +107,7 @@ bindExample(contract, (): World => ({}), { entries.some((entry) => entry.identifier === placeOrderIdentifier) && entries.some((entry) => entry.identifier === getOrderIdentifier), ); - expect( - records.filter( - (entry) => - entry.udfType === "Action" || - entry.udfType === "HttpAction" || - !clientCallers.has(entry.caller), - ), - ).toHaveLength(workers); + expect(await scheduledRows(backend)).toHaveLength(workers); expect( records.find((entry) => entry.identifier === placeOrderIdentifier), ).toMatchObject({ udfType: "Mutation", caller: "HttpApi", error: null }); diff --git a/tests/native/read-models.committed-state-visible.test.ts b/tests/native/read-models.committed-state-visible.test.ts index c1a0dbc..55ba1d8 100644 --- a/tests/native/read-models.committed-state-visible.test.ts +++ b/tests/native/read-models.committed-state-visible.test.ts @@ -15,6 +15,7 @@ import { type QueryWatch, } from "../../harness/clients.js"; import { required } from "../../harness/native.js"; +import { scheduledRows } from "./scheduled-rows.js"; import { line, orderWorld, @@ -29,7 +30,7 @@ const anchor = specTest({ void anchor; // The order summary is written by PlaceOrder's own mutation, in step 9 of the pipeline. A client // subscribed to the summary list sees the row the moment the command commits, and the function log -// shows that nothing but the command and the subscription's query ran in between. +// identifies those calls. Admin reads of the scheduled-function tables check that no job exists. type Summaries = FunctionReturnType; type Response = FunctionReturnType; interface World { @@ -40,9 +41,6 @@ interface World { } const orderId = "order-1"; const lines = [line("sku-1", 2, 250), line("sku-2", 1, 40)]; -// A record a client caused: a call over HTTP, or a query or mutation over the socket. A scheduled -// function, a cron or an action is a worker. -const clientCallers = new Set(["HttpApi", "SyncWorker"]); bindExample(contract, (): World => ({}), { "a read model maintained by the {useCase} use case": async ( world, @@ -121,8 +119,7 @@ bindExample(contract, (): World => ({}), { }, ]); }, - // Every record in the function log from the command to the subscription's update: the command's - // own mutation and the subscription's query, both caused by the client. + // The log identifies the two client calls; the tables include delayed scheduled functions. "the number of workers, jobs or queues that ran is {workers}": async ( world, { workers }, @@ -136,14 +133,7 @@ bindExample(contract, (): World => ({}), { (entry) => entry.identifier === "readModels:listOrderSummaries", ), ); - expect( - records.filter( - (entry) => - entry.udfType === "Action" || - entry.udfType === "HttpAction" || - !clientCallers.has(entry.caller), - ), - ).toHaveLength(workers); + expect(await scheduledRows(backend)).toHaveLength(workers); expect( records.find((entry) => entry.identifier === "ordering:placeOrder"), ).toMatchObject({ udfType: "Mutation", caller: "HttpApi", error: null }); diff --git a/tests/native/scheduled-rows.test.ts b/tests/native/scheduled-rows.test.ts new file mode 100644 index 0000000..5b20bb3 --- /dev/null +++ b/tests/native/scheduled-rows.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from "vitest"; +import { api } from "../../fixture/convex/_generated/api.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend, measure } from "../../harness/native.js"; +test("admin reads a delayed scheduled function in the parent and its component", async () => { + const backend = await fixtureBackend(); + const ids = (await ordinaryClient(backend.url).mutation( + api.scheduledRows.schedule, + {}, + )) as { parent: string; component: string }; + for (const [component, id] of [ + [undefined, ids.parent], + ["annex", ids.component], + ] as const) { + const rows = await backend.admin.readTable( + "_scheduled_functions", + component === undefined ? {} : { component }, + ); + measure(component ?? "parent", JSON.stringify(rows)); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ _id: id, state: { kind: "pending" } }); + expect(rows[0]!.scheduledTime).toBeGreaterThan(Date.now()); + } +}); diff --git a/tests/native/scheduled-rows.ts b/tests/native/scheduled-rows.ts new file mode 100644 index 0000000..a224fb1 --- /dev/null +++ b/tests/native/scheduled-rows.ts @@ -0,0 +1,13 @@ +import type { Backend } from "../../harness/backend.js"; +// Every mount in example/convex/convex.config.ts, plus the parent. +export async function scheduledRows(backend: Backend) { + const rows = await Promise.all( + [undefined, "orders", "inventory"].map((component) => + backend.admin.readTable( + "_scheduled_functions", + component === undefined ? {} : { component }, + ), + ), + ); + return rows.flat(); +} From a848e228745d7523d91b27a99919bd3e7c1605ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:10:02 +0200 Subject: [PATCH 012/155] test: record clean native measurement checks From 069919173a800a613b6e75f082c8351f6ed2a085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:13:06 +0200 Subject: [PATCH 013/155] test: prove replacement of later business changes --- ...led-functions.probe-7-in-place-import.sdp.md | 1 + ...s-exclude-pending-scheduled-functions.sdp.md | 1 + .../facts/f13-transactions-have-limits.sdp.md | 1 + ...led-functions-table-shows-failed-runs.sdp.md | 1 + tests/native/scheduler-cases.ts | 17 +++++++++++++++++ tests/native/scheduler-restore.ts | 14 +++++++++++++- 6 files changed, 34 insertions(+), 1 deletion(-) diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md index b9008b7..fd3199c 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md @@ -28,3 +28,4 @@ Then the observation is {result: "documents replaced and schedules preserved"} - A kept pending row referenced by an imported `v.id("_scheduled_functions")` is canceled after import; the cancellation is expected to prevent that reaction, while the uncanceled row created after export still runs against restored data. - The first native run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` held the original bound values. - The uncanceled reaction is expected to commit no earlier than its recorded scheduled time; elapsed waiting time is recorded rather than bounded. +- Another ReceiveStock and PlaceOrder after export change both contexts and the parent receipts and order summaries; replacement must restore the exported documents exactly and remove these later business changes. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md index 19908e1..a934f88 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md @@ -33,3 +33,4 @@ The backup page excludes scheduled functions, code, configuration and environmen - schedulerReferences: a stored `v.id("_scheduled_functions")` imports into the fresh destination, validates as a function argument and on another document write, returns null from `ctx.db.system.get`, and `ctx.scheduler.cancel` succeeds without making a row; in place the same ID resolves to the kept row and cancellation prevents that pending reaction; an uncanceled row still executes against restored data (F12, Probe 7) - keptReactions: pending mutations created after export survive import in place and execute at their scheduled time against the restored data; a failed row created after export also survives while its document reference does not, so the system table alone does not say which intent belongs to the exported data (F12, D13, D19, Probe 7) - statusScope: local CLI export and replacement are observed at the native backend tier; documented status remains because hosted dashboard backup and restore are not run, and the local observation is not a confirmed defect in that path (F12, Probe 7) +- cleanNativeEvidence: native backend `precompiled-2026-09-28-5c7cb5b`, SHA-256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae`, `npx vitest run --project native tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts tests/native/snapshot-replacement.test.ts`, `evidence/runs/native-20261002T210938Z-8e603b7-a3ef2d06-9562-4bb0-9963-c0b20231532f.json`, commit `8e603b748d6f95aecccfc15a1f9629b0756ec195`, clean true, all six native tests passed; earlier dirty records above retain exploratory expectation history and are not the evidence for the final behavior claims (Probe 7) diff --git a/design/specs/facts/f13-transactions-have-limits.sdp.md b/design/specs/facts/f13-transactions-have-limits.sdp.md index 7572fea..efa2aad 100644 --- a/design/specs/facts/f13-transactions-have-limits.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.sdp.md @@ -40,3 +40,4 @@ The documented numbers bound batches. Probe 4 observes transaction budgets; Prob - limitsAreCeilings: platform limits are ceilings, not batch sizes; every bulk operation, sweeper and batch states a bound tested against the pinned Convex version (D19, F13) - schedulingObservation: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, `npx vitest run --project native tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts --testTimeout 300000` as part of `evidence/runs/native-20261002T205553Z-592f1dc-97c74164-c3a8-4818-9f77-2f04b1456261.json`; the original 4 MiB hard-limit expectation failed; values are constructed inside the mutation, and the caught `runAfter` refusal returns normally with the number of earlier successful schedules, distinguishing this refusal from commit (F13, Probe 7) - schedulingBytes: native backend `precompiled-2026-09-28-5c7cb5b`, `npx vitest run --project native tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts` as part of `evidence/runs/native-20261002T205857Z-592f1dc-194ec93d-d96c-4eb7-b639-f7e2c630e648.json`, passed exact-byte boundaries: one string, multibyte text, 1024 strings and eight calls each fit at 16777216 accounted bytes and refuse their first larger tested value; for the tested `{ payload: string }` shape accounting is UTF-8 content bytes plus 14 per call, and for `{ payload: string[] }` it is content bytes plus 14 plus two per element per call; three newline bytes count as three, unlike six JSON escape bytes, so this is Convex value accounting, not JSON text length (F13, Probe 7) +- cleanNativeEvidence: native backend `precompiled-2026-09-28-5c7cb5b`, SHA-256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae`, `npx vitest run --project native tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts tests/native/snapshot-replacement.test.ts`, `evidence/runs/native-20261002T210938Z-8e603b7-a3ef2d06-9562-4bb0-9963-c0b20231532f.json`, commit `8e603b748d6f95aecccfc15a1f9629b0756ec195`, clean true, all six native tests passed; earlier dirty records above retain exploratory expectation history and are not the evidence for the final behavior claims (Probe 7) diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md index 41adbe6..2f12191 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md @@ -33,3 +33,4 @@ The scheduled-functions page documents five states and a seven-day completion-re - scanFinding: the first native scan expected filtered-out scheduler rows to consume the 32000-document allowance; at 1000 rows it instead counted zero document reads and zero bytes, while `databaseQueries.used` grew by 1001; a subsequent run at 32000 rows failed with "Your request timed out performing too many system operations." in `evidence/runs/native-20261002T205553Z-592f1dc-97c74164-c3a8-4818-9f77-2f04b1456261.json`; this is an observed system-operation refusal, not evidence that 32000 is the scan boundary (F16, F13, Probe 7) - scanBoundary: native backend `precompiled-2026-09-28-5c7cb5b`, `npx vitest run --project native tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts` as part of `evidence/runs/native-20261002T205857Z-592f1dc-194ec93d-d96c-4eb7-b639-f7e2c630e648.json`, observed 4091 rows scanning successfully and 4092 refused with "Too many reads in a single function execution (limit: 4096)"; bytes and documents read remain zero, but the scan adds N + 1 database queries to four already used; 32000 and 32001 are populated and refused, so the document ceiling is not the first boundary; the run failed because the expectation named only the system-operation timeout, and the example is rebound to the observed read refusal (F16, F13, Probe 7) - scanConfirmation: `npx vitest run --project native tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts`, native backend `precompiled-2026-09-28-5c7cb5b`, `evidence/runs/native-20261002T210145Z-592f1dc-1f503d30-03eb-431b-b48b-e946c2866bd5.json`, passed both the instrumented and plain scan at 4091 total rows and refused both at 4092; three rows are terminal, so these correspond to 4088 and 4089 outstanding rows; the admin count also confirms 32001 total rows (F16, F13, Probe 7) +- cleanNativeEvidence: native backend `precompiled-2026-09-28-5c7cb5b`, SHA-256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae`, `npx vitest run --project native tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts tests/native/snapshot-replacement.test.ts`, `evidence/runs/native-20261002T210938Z-8e603b7-a3ef2d06-9562-4bb0-9963-c0b20231532f.json`, commit `8e603b748d6f95aecccfc15a1f9629b0756ec195`, clean true, all six native tests passed; earlier dirty records above retain exploratory expectation history and are not the evidence for the final behavior claims (Probe 7) diff --git a/tests/native/scheduler-cases.ts b/tests/native/scheduler-cases.ts index e188ff8..ea516db 100644 --- a/tests/native/scheduler-cases.ts +++ b/tests/native/scheduler-cases.ts @@ -314,6 +314,23 @@ async function productionData(backend: Backend) { }, }); } +export async function placeAdditionalOrder(backend: Backend) { + const client = ordinaryClient(backend.url, { + token: await backend.issuer.token(subject), + }); + await client.mutation(api.receiving.receiveStock, { + tenantId: "t-1", + input: { items: [{ stockItemId: "sku", quantity: 2 }] }, + }); + await client.mutation(api.ordering.placeOrder, { + tenantId: "t-1", + requestKey: "another-order", + input: { + orderId: "another-order", + lines: [{ stockItemId: "sku", quantity: 1, unitPrice: 10 }], + }, + }); +} export async function exportData( backend: Backend, directory: string, diff --git a/tests/native/scheduler-restore.ts b/tests/native/scheduler-restore.ts index 556a436..a990727 100644 --- a/tests/native/scheduler-restore.ts +++ b/tests/native/scheduler-restore.ts @@ -1,7 +1,7 @@ import { expect } from "vitest"; import type { Backend } from "../../harness/backend.js"; import { measure } from "../../harness/native.js"; -import { exportData } from "./scheduler-cases.js"; +import { exportData, placeAdditionalOrder } from "./scheduler-cases.js"; import { schedulingBackend, schedulerRows, @@ -66,6 +66,7 @@ export async function freshImport(backend: Backend, directory: string) { export async function inPlaceImport(backend: Backend, directory: string) { const due = Date.now() + 45000; const exported = await exportData(backend, directory, due); + await placeAdditionalOrder(backend); for (const component of scopes) { const rows = await backend.admin.readTable( "schedulerData", @@ -110,6 +111,17 @@ export async function inPlaceImport(backend: Backend, directory: string) { ]); await backend.admin.setEnvironment({ SCHEDULER_VALUE: "after-export" }); const environment = await backend.admin.environment(); + const changedData = await dataRows(backend); + record("in-place data before import", changedData); + for (const table of [ + "orders/streams", + "orders/events", + "inventory/streams", + "inventory/events", + "parent/receipts", + "parent/orderSummaries", + ]) + expect(changedData[table]).not.toEqual(exported.data[table]); const before = await schedulerRows(backend); record("in-place scheduler tables before import", before); await backend.admin.replaceSnapshot(exported.path); From b58fe015dddc61b1e43584bb74f27f511e78e783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:17:29 +0200 Subject: [PATCH 014/155] test: establish migrations batch and context cursor boundaries --- design/README.md | 16 +- ...01-serializable-mutations-under-occ.sdp.md | 2 + ...ackfill.probe-6-batch-races-command.sdp.md | 34 +++ ...ckfill.probe-6-context-batch-cursor.sdp.md | 32 ++ ...ackfill.probe-6-context-enumeration.sdp.md | 35 +++ ...ill.probe-6-interruption-and-resume.sdp.md | 30 ++ ...migrations-fits-generation-backfill.sdp.md | 31 +- design/specs/facts/probe-plan.sdp.md | 6 +- fixture/convex/_generated/api.d.ts | 3 + fixture/convex/convex.config.ts | 2 + fixture/convex/depot/_generated/api.ts | 6 +- fixture/convex/depot/_generated/component.ts | 9 + fixture/convex/depot/convex.config.ts | 6 +- fixture/convex/depot/migrations.ts | 60 ++++ fixture/convex/depot/schema.ts | 13 +- fixture/convex/depotCommands.ts | 6 + fixture/convex/migrations.ts | 285 ++++++++++++++++++ fixture/convex/schema.ts | 10 + ...ckfill.probe-6-batch-races-command.test.ts | 156 ++++++++++ ...kfill.probe-6-context-batch-cursor.test.ts | 116 +++++++ ...ckfill.probe-6-context-enumeration.test.ts | 180 +++++++++++ ...ll.probe-6-interruption-and-resume.test.ts | 131 ++++++++ tests/native/migration-steps.ts | 117 +++++++ 23 files changed, 1262 insertions(+), 24 deletions(-) create mode 100644 design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.sdp.md create mode 100644 design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.sdp.md create mode 100644 design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md create mode 100644 design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.sdp.md create mode 100644 fixture/convex/depot/migrations.ts create mode 100644 fixture/convex/migrations.ts create mode 100644 tests/native/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.test.ts create mode 100644 tests/native/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.test.ts create mode 100644 tests/native/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.test.ts create mode 100644 tests/native/f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.test.ts create mode 100644 tests/native/migration-steps.ts diff --git a/design/README.md b/design/README.md index 96361e1..a944410 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -202 specs · 5 packs · 101 anchors → 308 nodes · 1097 edges (0 errors, 0 warnings) +206 specs · 5 packs · 105 anchors → 316 nodes · 1110 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -91,7 +91,7 @@ Detail follows the build (decision method, rule 4). Full means interfaces, TypeS | 5 | `agents` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 7 | | 6 | `advanced` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 8 | -Counts by kind and by stated readiness are derived, not kept here: `npx sdp census` writes the census, and the session check prints the readiness tally. Two groups state `scoped`: every Layer 4 to 6 non-example Spec, held by its deferral, and the assumed fact F17, held by its probe. Nothing states `ready`. Every pack carries `modelRefs: [spec:platform.vocabulary]`; the application pack also refs the example domain's model. +Counts by kind and by stated readiness are derived, not kept here: `npx sdp census` writes the census, and the session check prints the readiness tally. Every Layer 4 to 6 non-example Spec states `scoped`, held by its deferral; F17 states `defined` with its native Probe 6 evidence. Nothing states `ready`. Every pack carries `modelRefs: [spec:platform.vocabulary]`; the application pack also refs the example domain's model. What each layer fixes, in one paragraph each. @@ -236,7 +236,7 @@ An extension is a design claim the doc does not make. Each one is three things t ## Open questions for the owner -A `[blocking]` question in this corpus is one of two things. A deferral names the trigger that brings a Layer 4 to 6 Spec back. An assumed fact (F17) names the probe that would settle it. Every other open question is `[non-blocking]`, which means the design took a provisional reading and says so where it did. The rulings that matter most, grouped by where they come from. +A `[blocking]` question in this corpus is one of two things. A deferral names the trigger that brings a Layer 4 to 6 Spec back. An assumed fact names the probe that would settle it; F17 is probed by the four native Probe 6 examples. Every other open question is `[non-blocking]`, which means the design took a provisional reading and says so where it did. The rulings that matter most, grouped by where they come from. The doc's own open questions: @@ -270,7 +270,7 @@ Beyond the plan's list, three product decisions the packages surfaced: - The late-payment policy of `StartCheckout`: reallocate, ask a person, or refund as a separate operation ([`start-checkout-example`](specs/processes/start-checkout-example.sdp.md)). - Whether an old process run is kept, migrated or blocked on a given deploy ([`workflow-processes`](specs/processes/workflow-processes.sdp.md)). -Probes 1 to 5 ran on a native backend in slice S0 on 2026-10-01. Each result is recorded on its fact and on the Specs that rest on it, and [`probe-plan`](specs/facts/probe-plan.sdp.md) says what stays open: Probe 1 (the client guarantee holds across a restart, a closed client can lose a command, and the HTTP client does not retry, D6), Probe 2 (`ConvexError` data survives a nested mutation and a component boundary, F14, D7), Probe 3 (a component call costs about 2 ms more than a helper call on a local backend, with hosted cost and quota open, F4, D2, D8), Probe 4 (transaction limits are one budget across nested calls and components, F13, D10), Probe 5 (a parent query over a component query stays reactive and pinned pages stay contiguous, and the client hook loses rows after a capped split, F15, D8). Two are pending: Probe 6 (a backfill batch racing a live command; the migrations component, F17, D9) and Probe 7 (what a restore leaves of scheduler, Workpool and Workflow state; the system table's states and retention, F12, F16, D13, D19). +Probes 1 to 5 ran on a native backend in slice S0 on 2026-10-01. Each result is recorded on its fact and on the Specs that rest on it, and [`probe-plan`](specs/facts/probe-plan.sdp.md) says what stays open: Probe 1 (the client guarantee holds across a restart, a closed client can lose a command, and the HTTP client does not retry, D6), Probe 2 (`ConvexError` data survives a nested mutation and a component boundary, F14, D7), Probe 3 (a component call costs about 2 ms more than a helper call on a local backend, with hosted cost and quota open, F4, D2, D8), Probe 4 (transaction limits are one budget across nested calls and components, F13, D10), Probe 5 (a parent query over a component query stays reactive and pinned pages stay contiguous, and the client hook loses rows after a capped split, F15, D8). Probe 6 has native evidence on backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6 for the row race, interruption and resume, table scope and a parent batch that returns a context cursor to the driver. Probe 7 remains pending (what a restore leaves of scheduler, Workpool and Workflow state; the system table's states and retention, F12, F16, D13, D19). Every open question in the corpus, by Spec, is recipe 20 of the Protocol's catalog, the open-question register. It lists the Specs that hold a blocking question first and reports the totals, so no table is kept here: @@ -316,8 +316,12 @@ The Layer 2 design added two more for the same reason, and nine examples that ve | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split` | Probe 5: a full first page under a row cap of twice the page size is not split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap` | Probe 5: the same page under a row cap equal to the page size is split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap` | Probe 5: a parent list's page outgrows its row cap | +| none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-batch-races-command` | Probe 6: a migration batch races a live command | +| none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume` | Probe 6: a migration resumes after interruption | +| none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-context-enumeration` | Probe 6: migration table scope and context enumeration | +| none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor` | Probe 6: a parent batch returns a context cursor to the migrations driver | -Package A therefore has 79 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. +Package A therefore has 83 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. ## Integration notes @@ -376,7 +380,7 @@ Adopted from the Protocol on 2026-10-01, at the commit `package.json` pins: - Generics are written with real angle brackets inside inline code. The Unicode brackets the carrier once forced, 248 pairs on 114 lines of 29 Specs, are gone, so every pinned declaration can be pasted into source. - An unbound example below `ready` no longer warns. The expected validate verdict is 0 errors and 0 warnings. -- The `ready` floor reads `constrainedBy` and `decidedBy` as well as `refines` and `dependsOn`. The assumed fact F17 states `scoped` behind a blocking question that names its probe, so no Spec resting on it can state `ready` until the probe has run; F14 and F15 did the same until their probes ran in slice S0. F16 stays `defined`: its states and retention are documented. +- The `ready` floor reads `constrainedBy` and `decidedBy` as well as `refines` and `dependsOn`. F17 states `defined` with native Probe 6 evidence for the pinned releases; F14 and F15 previously stated `scoped` until their probes ran in slice S0. F16 stays `defined`: its states and retention are documented. - A deferral is a blocking open question that names its trigger, on the nine Layer 4 to 6 Specs. - The census sentence and the open-question table are no longer kept by hand. `sdp census` and recipe 20 derive them. - The extension register stays in this file. It is this project's policy, and `design/tools/check.py` keeps it honest. diff --git a/design/specs/facts/f01-serializable-mutations-under-occ.sdp.md b/design/specs/facts/f01-serializable-mutations-under-occ.sdp.md index e2c864d..b403c82 100644 --- a/design/specs/facts/f01-serializable-mutations-under-occ.sdp.md +++ b/design/specs/facts/f01-serializable-mutations-under-occ.sdp.md @@ -12,6 +12,8 @@ F1 · Status: documented · Doc status: Documented, S1 · Decisions: D1, D9. Convex runs every mutation as a serializable transaction and resolves conflicts by retrying the loser under optimistic concurrency control. The design rests on it twice: one mutation per business operation needs no lock protocol beside it, and a rebuild backfill can run online because each batch is ordered against live commands by the same mechanism. An engine retry caused by a conflict is invisible to the caller and is distinct from the logical version conflict the persistence adapter reports. +Probe 6 on the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6 races a component-driven batch with a live depot amendment in six trials: both summary generations end at source stream version 2; the first evidence records five conflict entries and five requested retries, as sizes rather than assertions, and checks execution overlap. F1 remains documented. + ## Intent - outcome: Record that mutations are serializable under optimistic concurrency, with its evidence status as a target, so that every design resting on it is found by its `constrainedBy` edge (F1) diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.sdp.md new file mode 100644 index 0000000..bd53e77 --- /dev/null +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.sdp.md @@ -0,0 +1,34 @@ +--- +id: spec:facts.f17-migrations-fits-generation-backfill.probe-6-batch-races-command +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f17-migrations-fits-generation-backfill + verifies: + - spec:facts.f17-migrations-fits-generation-backfill + - spec:facts.f01-serializable-mutations-under-occ +--- +# Probe 6: a migration batch races a live command + +Probe 6 · native backend tier · fixture composition. + +Native backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6. + +## Intent + +- outcome: The race reaches overlapping mutation calls; conflict and retry counts are recorded as sizes. Sequential checks cover each commit order. Hosted contention and throughput are not answered. (Probe 6, F17) + +```gwt +Given a document source at stream version 1 with its building summary missing +When a component-driven backfill races a live depot amendment in {trials: 6} trials +Then both generations end at stream version {version: 2} with the amended title +``` + +## Verification — executable + +- The bound values are expectations written before the first native run on the pinned releases. +- Each test owns its disposable backend; failure to reach the named boundary fails the example. +- The native backend evidence on the pinned releases records five conflict entries and five requested retries across six trials; both commit orders occur, every target finishes at version 2, and overlap is checked from execution timestamps. These counts are recorded sizes, not asserted limits. +- The missing building row forces the racing batch to insert a summary; the command updates that row if the batch commits first and skips it if the command commits first, after which the batch reads the newest source. +- The missing-row race records six conflicts and six requested retries in its first native evidence; separate serial checks bind both batch-first and command-first order to the same final stream version 2. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.sdp.md new file mode 100644 index 0000000..5141172 --- /dev/null +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.sdp.md @@ -0,0 +1,32 @@ +--- +id: spec:facts.f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f17-migrations-fits-generation-backfill + verifies: spec:facts.f17-migrations-fits-generation-backfill +--- +# Probe 6: a parent batch returns a context cursor to the migrations driver + +Probe 6 · native backend tier · fixture composition. + +Native backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6. + +## Intent + +- outcome: A parent internal mutation pages the context list and returns its cursor, processed count and completion flag to `runOne`, without `define` or a parent source table; its generation checkpoint and the component cursor commit with its rows. (Probe 6, F17) + +```gwt +Given {rows: 5} depot documents and a parent batch capped at {batch: 2} source rows +When the migrations driver cancels and resumes the context batch and a later batch throws after writing its checkpoint +Then resume writes {rows: 5} distinct summaries and both saved cursors agree at each interrupted boundary +``` + +## Verification — executable + +- The bound values are expectations written before the first native run on the pinned releases. +- The test reads the component's migrations table and the parent's generation row through admin access, checks the saved cursors at cancel and failure, and checks that a thrown batch rolls back its row writes and generation checkpoint while the component stores the error. +- Each batch makes one context list call, returns at most its bound, and writes no parent enumeration table; component-call counts are sizes recorded in evidence. +- Hosted quotas, throughput, backup replacement and projection coverage under changing tenants are not answered by this local example. +- The pinned native backend preserves equal saved cursors at cancel and failure, rolls the failed checkpoint and row writes back, and finishes five rows in three committed batches with no parent enumeration rows; the component reports success while the generation remains building. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md new file mode 100644 index 0000000..3f29d11 --- /dev/null +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md @@ -0,0 +1,35 @@ +--- +id: spec:facts.f17-migrations-fits-generation-backfill.probe-6-context-enumeration +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f17-migrations-fits-generation-backfill + verifies: spec:facts.f17-migrations-fits-generation-backfill +--- +# Probe 6: migration table scope and context enumeration + +Probe 6 · native backend tier · fixture composition. + +Native backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6. + +## Intent + +- outcome: The parent cannot name the context table in its typed data model. A runtime table walk stays in the parent. A depot migration can walk streams but direct parent-table writes fail; a passed parent function handle is tested separately. A parent migration over page records can call list, with two cursors and its page size bounded explicitly. Hosted quotas are not answered. (Probe 6, F17) + +```gwt +Given {rows: 5} depot documents and migrations mounted in the parent and depot +When parent and depot migrations walk tables and a parent migration calls the context list +Then the parent table walk fails with {parentRows: 0} processed context rows and the depot table walk sees {contextRows: 5} rows +``` + +## Verification — executable + +- The bound values are expectations written before the first native run on the pinned releases. +- Each test owns its disposable backend; failure to reach the named boundary fails the example. +- The first native run on the pinned releases expected an empty successful parent walk; it instead reported `failed` with `Uncaught Error: Index streams.by_creation_time not found.` The bound parent outcome is that observed refusal. +- A parent definition without the schema option is expected to reach the backend and complete with zero processed rows, while depot still holds all five sources; this value is bound before that form is run. +- The pinned native backend completes the depot walk with five visits, the depot callback with five parent rows, and the parent table-of-pages walk with three page records for five rows; the table-of-pages form keeps two cursors and requires migration batch size one for one bounded context page per batch. +- Before its native run, a parent definition with `customRange` but no schema option is bound to fail with `You must provide your schema to use a custom range.` +- The first native attempt without a schema option also reported `failed` with `Uncaught Error: Index streams.by_creation_time not found.`, rather than the expected successful zero-row walk; that form is rebound to the observed refusal. +- The pinned native backend returns `Uncaught Error: You must provide your schema to use a custom range.` for the no-schema range definition, as bound. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.sdp.md new file mode 100644 index 0000000..d6f78f8 --- /dev/null +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.sdp.md @@ -0,0 +1,30 @@ +--- +id: spec:facts.f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f17-migrations-fits-generation-backfill + verifies: spec:facts.f17-migrations-fits-generation-backfill +--- +# Probe 6: a migration resumes after interruption + +Probe 6 · native backend tier · fixture composition. + +Native backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6. + +## Intent + +- outcome: Cancel preserves the committed cursor, resume continues it, and a thrown batch preserves its preceding cursor and rolls back its row writes. A restart on the same storage preserves pending work. A backup replacement and hosted failures are not answered. (Probe 6, F17) + +```gwt +Given {rows: 12} document summaries and a migration with batch size {batch: 2} +When the migration is canceled, resumed, failed inside a batch and resumed, then the backend restarts with a batch pending +Then every source has exactly {copies: 1} target row and a successful visit after each completed migration +``` + +## Verification — executable + +- The bound values are expectations written before the first native run on the pinned releases. +- Each test owns its disposable backend; failure to reach the named boundary fails the example. +- On the pinned native backend, the values held: cancel at two processed rows, failed batch at the same cursor with its writes absent, resume at twelve rows with one committed visit each, and a pending worker finishing after a process kill and restart on the same storage. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md index c7c8883..1a5465d 100644 --- a/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md @@ -2,28 +2,41 @@ id: spec:facts.f17-migrations-fits-generation-backfill kind: constraint altitude: story -readiness: scoped +readiness: defined relations: refines: spec:facts.fact-ledger --- # The migrations component fits a generation backfill -F17 · Status: assumed · Doc status: Assumed · Decisions: D9. +F17 · Status: probed · Doc status: Assumed · Decisions: D9. -Online rebuild registers a new generation and backfills it in batches that read or fold each source at its current stream version and write the row only if the target is missing or older. The migrations component provides resumable batching, which the product page states. Whether a batch of it races a live command the way D9 needs, and whether its batch size can be set below the transaction limits, is not stated, so the fact stays assumed and Probe 6 stands. +Native backend evidence is bound to `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6. A parent-table migration races a live depot amendment without overwriting the newest summary. Cancel and resume preserve the component cursor, a thrown batch rolls back its rows while the component commits a failed status, and a backend restart on the same storage resumes pending work. + +The same native backend runs a migration mounted inside depot over depot's streams. It cannot directly write a parent summary by ID; it can call a supplied parent function handle, at one parent call per source row. A parent `define` cannot name depot's table through the parent's typed data model. For context enumeration, a parent internal mutation can call the context's `list` once, write the summary rows and generation checkpoint, and return `continueCursor`, `processed` and `isDone` to `runOne` without `define`. This form uses the context cursor in both saved checkpoints and no parent enumeration table. It keeps the tested two-row batch bound; the rebuild's byte bounds remain the batch author's duty. + +On the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6, the first parent-table attempt expected an empty walk but reported `failed` with `Uncaught Error: Index streams.by_creation_time not found.` The parent schema has no context table. The first attempt without a schema option also expected a successful empty walk and returned the same failed status and exact error instead. A parent definition with `customRange` and no schema option returned `Uncaught Error: You must provide your schema to use a custom range.` as expected. (Probe 6) ## Intent -- outcome: Record that `@convex-dev/migrations` fits a generation backfill, with its evidence status as a target, so that every design resting on it is found by its `constrainedBy` edge (F17) -- assumption: The fact is assumed, not documented; every Spec that rests on it names this assumption and the probe plan carries the probe (F17, Probe 6) +- outcome: Record the native backend evidence for parent-table batches, context enumeration, interruption and checkpoint ownership, so designs resting on the migrations component distinguish its driver from its table helper. (F17, Probe 6) +- risk: In the compiled tier with these package pins and `exactOptionalPropertyTypes`, `runOne` refuses the reference generated by `define` because `_args.batchSize` includes `undefined`; the fixture adapts the function reference explicitly to reach the native backend. (F17, Probe 6) ### Open questions -- [blocking] Probe 6 pending: until it runs on a native backend this fact stays assumed, and the decisions it serves (D9) rest on the author's reading of the pages (F17, Probe 6) +- [non-blocking] Hosted quotas, sustained contention, automatic stall detection, backup replacement, old encrypted cursors, fence advancement, parallel callbacks and a rebuild across changing tenants are not run by these examples; the native backend evidence proves only the named boundaries and does not settle those behaviors. (F17, Probe 6) ## Constraints -- statement: `@convex-dev/migrations` fits a generation backfill (F17) +- statement: `@convex-dev/migrations` 0.3.6 drives parent-table backfill and a parent internal mutation that pages a context query and returns a migration result; `define` alone does not enumerate another component's table. (F17, Probe 6) - flavor: convex-fact -- target: evidence.status:assumed -- measurableBy: the component's product page states batches with saved progress, resume from the last checkpoint after a timeout, and running over all documents or a subset; batch size configuration is not on the page; doc status Assumed; Probe 6 (F17, D9) +- target: evidence.status:probed +- measurableBy: Four Probe 6 example Specs bind exact steps to native tests on the pinned releases: the row race, interruption and resume, table scope and context enumeration, and the parent batch returning a context cursor. (F17, D9) + +## Design + +- raceEvidence: Six native backend trials end both generations at stream version 2; the first evidence records five conflict entries and five requested retries, each conflict's zero-based `retryCount` at 0; the missing-row evidence records six conflicts and six requested retries; these counts are sizes, not asserted limits, and execution overlap is required. (F17, F1, Probe 6) +- interruptionEvidence: With twelve sources and batch size two, cancel leaves two processed rows and a saved cursor; a later batch throwing after a write leaves those two rows and that cursor, with `state: failed`; resume reaches twelve distinct rows and one committed visit each; a pending batch survives a process kill and restart on the same storage. (F17, Probe 6) +- stateOwner: The migrations component's `migrations` table stores `name`, `cursor`, `isDone`, `workerId`, `error`, `processed`, `latestStart` and `latestEnd`; `getStatus` derives its state from that row and its `_scheduled_functions` worker, and returns a null cursor on success while the stored completion cursor is `[]`. (F17, Probe 6) +- checkpointEvidence: A native parent context batch writes the generation cursor and rows in its callback, then the component saves the same cursor in the enclosing mutation; throwing after the generation write rolls both parent writes back, while the component catches the error and saves failure against its unchanged cursor; a successful driver status leaves the generation `building`, since verification and activation are separate. (F17, Probe 6) +- enumerationCost: The native parent context batch and the parent-table-of-pages form each call `list` once per nonempty batch and cross two component boundaries per scheduled batch that enumerates a context page, driver to parent and parent to context; the table-of-pages form has a second cursor and needs migration batch size one to bound its total source rows by one list page; a context-table callback makes one parent call per source row. (F17, Probe 6) +- sourceEvidence: In `node_modules/@convex-dev/migrations/dist/client/index.js`, `define` pages its local database at lines 273 to 298, calls `migrateOne` at lines 310 to 329, and forms the batch result at lines 330 to 334 and returns it at line 369; `runOne` dispatches the supplied function handle at lines 407 to 423; in `dist/component/lib.js`, lines 79 to 86 run the batch, lines 92 to 97 schedule its successor, lines 136 to 155 catch failure, and line 157 saves state. (F17, Probe 6) diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index b83e22d..a963de0 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -20,7 +20,7 @@ relations: Feature · Detail: full transcription · Traces: Probe 1 to Probe 7, F4, F5, F12, F13, F14, F15, F16, F17, S13, E-16. -Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`, so Probe 7 need only confirm them and answer what a restore leaves; the backup page says nothing about component data or the scheduler, so that half of Probe 7 stands. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, and each result is recorded on its fact; probes 6 and 7 have not run. +Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`, so Probe 7 need only confirm them and answer what a restore leaves; the backup page says nothing about component data or the scheduler, so that half of Probe 7 stands. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, and each result is recorded on its fact; Probe 6 runs on native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6, with its four examples and results on F17; Probe 7 has not run. The probes are not the first experiment. The experiment builds Layers 0 to 2 and measures cost; the probes settle single facts and can run before it, in a fixture app that is separate from the example app. @@ -38,7 +38,7 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - [non-blocking] Probe 1 ran on 2026-10-01: a closed client leaves its mutation executed once or not at all, a backend restart leaves it executed once, and the HTTP client does not retry; whether callers of `ConvexHttpClient` must supply a request key is D6's question for the owner (Probe 1, D6) - [non-blocking] Probe 3 ran on 2026-10-01 for latency on a local backend; a local backend has no function-call quota, so that half waits for a hosted deployment (Probe 3, F4, D2, D8) - [non-blocking] Probe 5 ran on 2026-10-01: a parent query over a component query stays reactive, pages pinned by their end cursors stay contiguous, and a capped page crosses the boundary as `SplitRequired`; the `convex-helpers/react` hook then lost rows, and slice S2 decides how a context list is capped before it builds one (Probe 5, F15, S4, D8) -- [non-blocking] Probe 6 pending: it must show a backfill batch racing a live command under optimistic concurrency, and that the migrations component fits a generation backfill (Probe 6, F17, D9) +- [non-blocking] Probe 6 records native backend evidence on the pins named by F17: a row race, cancel and resume, thrown-batch rollback, restart, table scope and a parent batch returning a context cursor to the migrations driver; hosted quotas and sustained contention are not run. (Probe 6, F17, D9) - [non-blocking] Probe 7 pending: before Layer 3, it must show what a restore leaves of Workpool and Workflow state and confirm the states and retention of `_scheduled_functions` on a native backend (Probe 7, F12, F16, D13, D19) - [non-blocking] Extension E-16: the doc says each probe is one small test on a native backend and that a probed fact is one a native backend test showed, and does not say how a test that settles an unknown states what it expects or what follows when the backend answers otherwise; the option taken here: a probe is written as example Specs that verify its fact, each bound to its native test; an example's bound values are the expectation written before the first run; a first run that shows another value is a finding recorded on the fact, and the example is then bound to what that release does and says so; an example fails when its run did not reach the boundary it is about; a later run on a new release that shows another value is again a finding, and the test is not repaired to keep the old value; a size is recorded and not asserted, while a limit the backend enforces is bound as observed; and what a local backend cannot answer is named on the example and stays open on the fact; the owner confirms (E-16, Probes, Acceptance scenarios) @@ -60,7 +60,7 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - Probe 3 measures one component call from a parent mutation and from a parent query in latency and in function-call quota, and reports the difference from a plain helper; it serves D2 and D8 (Probe 3, F4) - Probe 4 reads and writes near the per-transaction limits from a parent, from a nested mutation and from a component, and records whether the limits add up across the boundaries or apply per call; it serves D10 (Probe 4, F13) - Probe 5 subscribes to a parent query that calls a component query, changes the component's data, and checks that the subscription updates; it then pages a component list built with `paginator` through the parent while live writes land inside the paged range, with the client passing the end cursor as `convex-helpers` describes, records whether the pages stay contiguous, and fills one page's range past the component's `maximumRowsRead` so that a re-run comes back `SplitRequired`, recording whether the `convex-helpers/react` hook splits it across the boundary without a gap; it serves D8 (Probe 5, F15, S4) -- Probe 6 runs a backfill batch through the migrations component while a live command writes the same read-model row, and checks that neither overwrites newer data and that the batch resumes after interruption; it serves D9 (Probe 6, F1, F17) +- Probe 6 binds four native backend examples on F17's pins: a component-driven batch racing a live depot command, cancel and resume with a thrown batch and process restart, parent and context table scope, and a parent batch returning a context list cursor to the migrations driver with its generation checkpoint in the same transaction; the results distinguish the table helper from the batch driver and serve D9. (Probe 6, F1, F17) - Probe 7, before Layer 3, takes a backup with pending scheduled functions, Workpool jobs and a Workflow run in flight, restores it, and records what survives; it reads `_scheduled_functions` for a failed run and confirms the states and the retention window; it serves D13 and D19 (Probe 7, F12, F16) ## Design diff --git a/fixture/convex/_generated/api.d.ts b/fixture/convex/_generated/api.d.ts index 694907b..ae33aa5 100644 --- a/fixture/convex/_generated/api.d.ts +++ b/fixture/convex/_generated/api.d.ts @@ -19,6 +19,7 @@ import type * as identity from "../identity.js"; import type * as limits from "../limits.js"; import type * as list from "../list.js"; import type * as markers from "../markers.js"; +import type * as migrations from "../migrations.js"; import type * as nonUiCaller from "../nonUiCaller.js"; import type * as notes from "../notes.js"; import type * as orders from "../orders.js"; @@ -49,6 +50,7 @@ declare const fullApi: ApiFromModules<{ limits: typeof limits; list: typeof list; markers: typeof markers; + migrations: typeof migrations; nonUiCaller: typeof nonUiCaller; notes: typeof notes; orders: typeof orders; @@ -89,6 +91,7 @@ export declare const internal: FilterApi< >; export declare const components: { + migrations: import("@convex-dev/migrations/_generated/component.js").ComponentApi<"migrations">; annex: import("../annex/_generated/component.js").ComponentApi<"annex">; depot: import("../depot/_generated/component.js").ComponentApi<"depot">; yard: import("../yard/_generated/component.js").ComponentApi<"yard">; diff --git a/fixture/convex/convex.config.ts b/fixture/convex/convex.config.ts index 69121e2..0b8461b 100644 --- a/fixture/convex/convex.config.ts +++ b/fixture/convex/convex.config.ts @@ -2,7 +2,9 @@ import { defineApp } from "convex/server"; import annex from "./annex/convex.config.js"; import depot from "./depot/convex.config.js"; import yard from "./yard/convex.config.js"; +import migrations from "@convex-dev/migrations/convex.config.js"; const app = defineApp(); +app.use(migrations); app.use(annex); // The fixture contexts. Each mount name is its contextId. The yard is there so a use case can call // two contexts. diff --git a/fixture/convex/depot/_generated/api.ts b/fixture/convex/depot/_generated/api.ts index 736e5ad..67b2541 100644 --- a/fixture/convex/depot/_generated/api.ts +++ b/fixture/convex/depot/_generated/api.ts @@ -8,6 +8,7 @@ * @module */ +import type * as migrations from "../migrations.js"; import type * as operations from "../operations.js"; import type * as queries_document from "../queries/document.js"; import type * as streams from "../streams.js"; @@ -20,6 +21,7 @@ import type { import { anyApi, componentsGeneric } from "convex/server"; const fullApi: ApiFromModules<{ + migrations: typeof migrations; operations: typeof operations; "queries/document": typeof queries_document; streams: typeof streams; @@ -51,4 +53,6 @@ export const internal: FilterApi< FunctionReference > = anyApi as any; -export const components = componentsGeneric() as unknown as {}; +export const components = componentsGeneric() as unknown as { + migrations: import("@convex-dev/migrations/_generated/component.js").ComponentApi<"migrations">; +}; diff --git a/fixture/convex/depot/_generated/component.ts b/fixture/convex/depot/_generated/component.ts index 3146b09..bfc088c 100644 --- a/fixture/convex/depot/_generated/component.ts +++ b/fixture/convex/depot/_generated/component.ts @@ -23,6 +23,15 @@ import type { FunctionReference } from "convex/server"; */ export type ComponentApi = { + migrations: { + configure: FunctionReference< + "mutation", + "internal", + { callback: string; parentId: string }, + any, + Name + >; + }; operations: { addStock: FunctionReference< "mutation", diff --git a/fixture/convex/depot/convex.config.ts b/fixture/convex/depot/convex.config.ts index eefbf9f..ced07f4 100644 --- a/fixture/convex/depot/convex.config.ts +++ b/fixture/convex/depot/convex.config.ts @@ -1,3 +1,5 @@ import { defineComponent } from "convex/server"; -// The mount name is the context's contextId, the constant in streams.ts. -export default defineComponent("depot"); +import migrations from "@convex-dev/migrations/convex.config.js"; +const depot = defineComponent("depot"); +depot.use(migrations); +export default depot; diff --git a/fixture/convex/depot/migrations.ts b/fixture/convex/depot/migrations.ts new file mode 100644 index 0000000..aac3070 --- /dev/null +++ b/fixture/convex/depot/migrations.ts @@ -0,0 +1,60 @@ +import { Migrations } from "@convex-dev/migrations"; +import { + type FunctionHandle, + type GenericDataModel, + type GenericMutationCtx, +} from "convex/server"; +import { v, type GenericId } from "convex/values"; +import { components } from "./_generated/api.js"; +import { internalMutation, mutation } from "./_generated/server.js"; +import schema from "./schema.js"; +const migrations = new Migrations(components.migrations, { + internalMutation, + schema, +}); +export const streams = migrations.define({ + table: "streams", + customRange: (q) => + q.withIndex("by_identity", (q) => + q.eq("tenantId", "tenant").eq("streamType", "document"), + ), + batchSize: 2, + migrateOne: async (ctx, row) => { + await ctx.db.insert("migrationVisits", { streamId: row.streamId }); + }, +}); +export const parentWrite = migrations.define({ + table: "streams", + batchSize: 1, + migrateOne: async (ctx) => { + const settings = (await ctx.db.query("migrationSettings").unique())!; + // @ts-expect-error The context owns no documentSummaries table. + const parentTable: import("./_generated/dataModel.js").TableNames = + "documentSummaries"; + void parentTable; + await (ctx as GenericMutationCtx).db.patch( + "documentSummaries", + settings.parentId as GenericId<"documentSummaries">, + { title: "context write" }, + ); + }, +}); +export const callback = migrations.define({ + table: "streams", + batchSize: 2, + migrateOne: async (ctx, row) => { + const settings = (await ctx.db.query("migrationSettings").unique())!; + await ctx.runMutation(settings.callback as FunctionHandle<"mutation">, { + tenantId: row.tenantId, + streamId: row.streamId, + version: row.streamVersion, + title: (row.state as { title: string }).title, + }); + }, +}); +export const configure = mutation({ + args: { parentId: v.string(), callback: v.string() }, + handler: async (ctx, args) => { + await ctx.db.insert("migrationSettings", args); + }, +}); diff --git a/fixture/convex/depot/schema.ts b/fixture/convex/depot/schema.ts index e88f586..fafe8b7 100644 --- a/fixture/convex/depot/schema.ts +++ b/fixture/convex/depot/schema.ts @@ -1,4 +1,11 @@ -import { defineSchema } from "convex/server"; +import { defineSchema, defineTable } from "convex/server"; import { contextTables } from "../../../src/context/index.js"; -// A context component's schema holds the context library's tables and nothing else. -export default defineSchema({ ...contextTables }); +import { v } from "convex/values"; +export default defineSchema({ + ...contextTables, + migrationVisits: defineTable({ streamId: v.string() }), + migrationSettings: defineTable({ + parentId: v.string(), + callback: v.string(), + }), +}); diff --git a/fixture/convex/depotCommands.ts b/fixture/convex/depotCommands.ts index 39d2996..c2c6480 100644 --- a/fixture/convex/depotCommands.ts +++ b/fixture/convex/depotCommands.ts @@ -359,3 +359,9 @@ const claimStockDeclaration = stockCommand( ); export const claimStock = publicCommand(claimStockDeclaration); export const claimStockInternal = internalCommand(claimStockDeclaration); + +export const amendSummarizedDocument = publicCommand({ + ...amendDocumentDeclaration, + name: "AmendSummarizedDocument", + readModels: [{ readModel: documentSummary, source: documentSource }], +}); diff --git a/fixture/convex/migrations.ts b/fixture/convex/migrations.ts new file mode 100644 index 0000000..cc805e0 --- /dev/null +++ b/fixture/convex/migrations.ts @@ -0,0 +1,285 @@ +import { + Migrations, + type MigrationStatus, + type MigrationFunctionReference, +} from "@convex-dev/migrations"; +import { + createFunctionHandle, + makeFunctionReference, + type GenericMutationCtx, + type GenericDataModel, +} from "convex/server"; +import { v, type GenericId } from "convex/values"; +import { components, internal } from "./_generated/api.js"; +import { + internalMutation, + internalQuery, + type MutationCtx, +} from "./_generated/server.js"; +import schema from "./schema.js"; +import { documentSummary } from "./summaries.js"; +import type { StreamVersion } from "../../src/kernel/index.js"; + +const migrations = new Migrations(components.migrations, { + internalMutation, + schema, +}); +// The package's runOne reference type omits explicit undefined on optional arguments. +const summaryMigration = makeFunctionReference( + "migrations:summaries", +) as unknown as MigrationFunctionReference; +const enumerationMigration = makeFunctionReference( + "migrations:enumeration", +) as unknown as MigrationFunctionReference; +type Document = Parameters[1]; + +async function writeSummary(ctx: MutationCtx, tenantId: string, dto: Document) { + const existing = await ctx.db + .query("documentSummaries") + .withIndex("by_key", (q) => + q.eq("tenantId", tenantId).eq("generation", 2).eq("key", dto.documentId), + ) + .unique(); + if (existing && existing.sourceVersions[0]!.version >= dto.version.version) + return; + const row = { + ...documentSummary.projection.project(tenantId, dto, [dto.version])!, + tenantId, + generation: 2, + key: dto.documentId, + projectionVersion: 1, + sourceVersions: [dto.version], + }; + if (existing) await ctx.db.replace("documentSummaries", existing._id, row); + else await ctx.db.insert("documentSummaries", row); +} +export const summaries = migrations.define({ + table: "documentSummaries", + customRange: (q) => + q.withIndex("by_key", (q) => + q.eq("tenantId", "tenant").eq("generation", 1), + ), + batchSize: 2, + migrateOne: async (ctx, row) => { + const dto = (await ctx.runQuery(components.depot.queries.document.get, { + tenantId: row.tenantId, + streamId: row.key, + })) as Document; + const settings = await ctx.db.query("migrationSettings").unique(); + // Repeated reads widen the overlap with a live command without changing its domain work. + for (let i = 0; i < (settings?.reads ?? 0); i++) + await ctx.db.get("documentSummaries", row._id); + await writeSummary(ctx, row.tenantId, dto); + await ctx.db.insert("migrationVisits", { + key: row.key, + version: dto.version.version, + }); + if (settings?.failKey === row.key) + throw new Error("Backfill batch interrupted after row write"); + }, +}); +export const run = internalMutation({ + args: { + cancel: v.optional(v.boolean()), + reset: v.optional(v.boolean()), + batchSize: v.optional(v.number()), + }, + handler: async ( + ctx, + args, + ): Promise<{ started: MigrationStatus; stopped: MigrationStatus | null }> => { + const started = await migrations.runOne(ctx, summaryMigration, { + ...(args.reset === undefined ? {} : { reset: args.reset }), + ...(args.batchSize === undefined ? {} : { batchSize: args.batchSize }), + }); + const stopped = args.cancel + ? await migrations.cancel(ctx, summaryMigration) + : null; + return { started, stopped }; + }, +}); +export const status = internalQuery({ + args: {}, + handler: (ctx): Promise => + migrations.getStatus(ctx, { migrations: [summaryMigration] }), +}); +export const configure = internalMutation({ + args: { + failKey: v.union(v.string(), v.null()), + reads: v.number(), + clear: v.boolean(), + }, + handler: async (ctx, { clear, ...settings }) => { + const old = await ctx.db.query("migrationSettings").unique(); + if (old) await ctx.db.replace("migrationSettings", old._id, settings); + else await ctx.db.insert("migrationSettings", settings); + if (clear) { + for (const row of await ctx.db.query("migrationVisits").collect()) + await ctx.db.delete("migrationVisits", row._id); + for (const row of await ctx.db.query("documentSummaries").collect()) + if (row.generation === 2) + await ctx.db.delete("documentSummaries", row._id); + } + }, +}); +// This declaration is deliberately refused by the parent's generated data model. +export const contextTable = migrations.define({ + // @ts-expect-error The parent owns no streams table. + table: "streams", + migrateOne: async (ctx, row) => { + await ctx.db.insert("migrationVisits", { key: row._id, version: 0 }); + }, +}); +export const contextId = internalMutation({ + args: { id: v.string() }, + handler: (ctx, { id }) => + (ctx as GenericMutationCtx).db.get( + "streams", + id as GenericId<"streams">, + ), +}); +export const enumeration = migrations.define({ + table: "enumerationPages", + batchSize: 1, + migrateOne: async (ctx, row) => { + const page = await ctx.runQuery(components.depot.queries.document.list, { + tenantId: row.tenantId, + paginationOpts: { cursor: row.cursor, numItems: row.size }, + }); + for (const value of page.page) + await writeSummary(ctx, row.tenantId, value as Document); + console.log("Enumeration batch", { + items: page.page.length, + contextCalls: 1, + isDone: page.isDone, + }); + if (!page.isDone) + await ctx.db.insert("enumerationPages", { + tenantId: row.tenantId, + cursor: page.continueCursor, + size: row.size, + }); + }, +}); +export const runEnumeration = internalMutation({ + args: {}, + handler: (ctx): Promise => + migrations.runOne(ctx, enumerationMigration), +}); +export const prepareContext = internalMutation({ + args: { parentId: v.string() }, + handler: async (ctx, { parentId }): Promise => + ctx.runMutation(components.depot.migrations.configure, { + parentId, + callback: await createFunctionHandle(internal.migrations.receive), + }), +}); +export const receive = internalMutation({ + args: { + tenantId: v.string(), + streamId: v.string(), + version: v.number(), + title: v.string(), + }, + handler: async (ctx, args) => { + const version: StreamVersion = { + tenantId: args.tenantId, + contextId: "depot", + streamType: "document", + streamId: args.streamId, + version: args.version, + }; + await writeSummary(ctx, args.tenantId, { + documentId: args.streamId, + title: args.title, + status: "draft", + amendments: 0, + version, + }); + }, +}); + +// A whole parent batch, with the context's cursor as the driver's cursor. +export const contextBatch = internalMutation({ + args: { + cursor: v.optional(v.union(v.string(), v.null())), + batchSize: v.optional(v.number()), + dryRun: v.optional(v.boolean()), + oneBatchOnly: v.optional(v.boolean()), + }, + handler: async ( + ctx, + args, + ): Promise => { + if (args.dryRun) + throw new Error("This fixture batch requires dryRun false"); + const size = Math.min(args.batchSize ?? 2, 2); + const page = await ctx.runQuery(components.depot.queries.document.list, { + tenantId: "tenant", + paginationOpts: { cursor: args.cursor ?? null, numItems: size }, + includeDeleted: true, + }); + const generation = (await ctx.db + .query("generations") + .withIndex("by_read_model", (q) => + q.eq("readModel", "documentSummary").eq("generation", 2), + ) + .unique())!; + const settings = await ctx.db.query("migrationSettings").unique(); + for (const value of page.page) { + const dto = value as Document; + await writeSummary(ctx, "tenant", dto); + await ctx.db.insert("migrationVisits", { + key: dto.documentId, + version: dto.version.version, + }); + } + await ctx.db.patch("generations", generation._id, { + cursor: { tenantId: "tenant", pageCursor: page.continueCursor }, + batchesDone: generation.batchesDone + 1, + rowsWritten: generation.rowsWritten + page.page.length, + }); + if ( + page.page.some( + (value) => (value as Document).documentId === settings?.failKey, + ) + ) + throw new Error("Context batch interrupted after checkpoint write"); + console.log("Context batch", { items: page.page.length, contextCalls: 1 }); + return { + continueCursor: page.continueCursor, + processed: page.page.length, + isDone: page.isDone, + }; + }, +}); +const contextBatchMigration = makeFunctionReference( + "migrations:contextBatch", +) as unknown as MigrationFunctionReference; +export const runContextBatch = internalMutation({ + args: { cancel: v.optional(v.boolean()), batchSize: v.number() }, + handler: async (ctx, args): Promise => { + const result = await migrations.runOne(ctx, contextBatchMigration, { + batchSize: args.batchSize, + }); + return args.cancel ? migrations.cancel(ctx, contextBatchMigration) : result; + }, +}); + +const tableWalker = new Migrations(components.migrations, { internalMutation }); +export const contextTableWithoutSchema = tableWalker.define({ + // @ts-expect-error The parent owns no streams table, even without the schema option. + table: "streams", + migrateOne: async (ctx, row) => { + await ctx.db.insert("migrationVisits", { key: row._id, version: 0 }); + }, +}); + +export const rangeWithoutSchema = tableWalker.define({ + table: "documentSummaries", + customRange: (q) => + q.withIndex("by_key", (q) => + q.eq("tenantId", "tenant").eq("generation", 1), + ), + migrateOne: () => undefined, +}); diff --git a/fixture/convex/schema.ts b/fixture/convex/schema.ts index eb5643c..96867b8 100644 --- a/fixture/convex/schema.ts +++ b/fixture/convex/schema.ts @@ -5,6 +5,16 @@ import { readModelTables, rowConventions } from "../../src/read-model/index.js"; import { documentTitleFields } from "./documentTitles.js"; import { documentSummaryFields } from "./summaries.js"; export default defineSchema({ + migrationSettings: defineTable({ + failKey: v.union(v.string(), v.null()), + reads: v.number(), + }), + migrationVisits: defineTable({ key: v.string(), version: v.number() }), + enumerationPages: defineTable({ + tenantId: v.string(), + cursor: v.union(v.string(), v.null()), + size: v.number(), + }), notes: defineTable({ source: v.string() }), depthRows: defineTable({ trial: v.string() }).index("by_trial", ["trial"]), markers: defineTable({ trial: v.string() }).index("by_trial", ["trial"]), diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.test.ts new file mode 100644 index 0000000..727daa3 --- /dev/null +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.test.ts @@ -0,0 +1,156 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe6BatchRacesCommandContract as contract } from "../../generated/contracts/facts.f17-migrations-fits-generation-backfill.probe-6-batch-races-command.contract.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f17-migrations-fits-generation-backfill.probe-6-batch-races-command", + ), + verifies: ref( + "spec:facts.f17-migrations-fits-generation-backfill.probe-6-batch-races-command", + ), +}); +void anchor; +import { setTimeout as sleep } from "node:timers/promises"; +import { api } from "../../fixture/convex/_generated/api.js"; +import { setup, record } from "./migration-steps.js"; +type Prepared = Awaited>; +interface World { + first?: Prepared; + results: number[]; +} +bindExample(contract, (): World => ({ results: [] }), { + "a document source at stream version 1 with its building summary missing": + async (world) => { + world.first = await setup(1); + }, + "a component-driven backfill races a live depot amendment in {trials} trials": + async (world, { trials }) => { + for (let trial = 0; trial < trials; trial++) { + const { backend, client } = trial === 0 ? world.first! : await setup(1); + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 400, + clear: false, + }); + const mark = await backend.admin.logMark(); + const batch = backend.admin.run("migrations:run", { + reset: true, + batchSize: 1, + }); + await sleep(trial % 2 === 0 ? 5 : 20); + const command = client.mutation( + api.depotCommands.amendSummarizedDocument, + { + tenantId: "tenant", + input: { documentId: "doc-00", title: "amended" }, + }, + ); + await Promise.all([batch, command]); + const rows = await backend.admin.readTable("documentSummaries"); + expect(await backend.admin.readTable("migrationVisits")).toHaveLength( + 1, + ); + expect(rows).toHaveLength(2); + for (const row of rows) { + expect(row.title).toBe("amended"); + world.results.push( + (row.sourceVersions as { version: number }[])[0]!.version, + ); + } + const logs = await backend.admin.completionsSince( + mark, + (entries) => + entries.some( + (entry) => + entry.identifier === "depotCommands:amendSummarizedDocument", + ) && entries.some((entry) => entry.identifier === "migrations:run"), + ); + const observed = logs as ((typeof logs)[number] & { + executionTimestamp: number; + willRetry: boolean; + occInfo: { retryCount: number; tableName: string } | null; + })[]; + const commandLog = observed.find( + (entry) => + entry.identifier === "depotCommands:amendSummarizedDocument", + )!; + const batchLogs = observed.filter( + (entry) => entry.identifier === "migrations:run", + ); + const overlapped = batchLogs.some( + (entry) => + entry.executionTimestamp < + commandLog.executionTimestamp + commandLog.executionTime && + commandLog.executionTimestamp < + entry.executionTimestamp + entry.executionTime, + ); + record(`race ${trial}`, { + rows, + entries: logs, + overlapped, + conflicts: observed.filter((entry) => entry.occInfo !== null).length, + retryRequests: observed.filter((entry) => entry.willRetry).length, + }); + expect(overlapped, "The mutation executions must overlap").toBe(true); + // Both serial orders must also preserve the newest version. + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 0, + clear: false, + }); + await backend.admin.run("migrations:run", { reset: true }); + expect( + (await backend.admin.readTable("documentSummaries")).map( + (row) => row.title, + ), + ).toEqual(["amended", "amended"]); + } + for (const first of ["batch", "command"] as const) { + const { backend, client } = await setup(1); + const batch = () => backend.admin.run("migrations:run", {}); + const command = () => + client.mutation(api.depotCommands.amendSummarizedDocument, { + tenantId: "tenant", + input: { documentId: "doc-00", title: "amended" }, + }); + if (first === "batch") { + await batch(); + expect( + (await backend.admin.readTable("documentSummaries")).filter( + (row) => row.generation === 2, + ), + ).toHaveLength(1); + await command(); + } else { + await command(); + expect( + (await backend.admin.readTable("documentSummaries")).filter( + (row) => row.generation === 2, + ), + ).toHaveLength(0); + await batch(); + } + const rows = await backend.admin.readTable("documentSummaries"); + expect(rows).toHaveLength(2); + for (const row of rows) { + expect(row.title).toBe("amended"); + world.results.push( + (row.sourceVersions as { version: number }[])[0]!.version, + ); + } + record(`serial ${first} first`, rows); + } + }, + "both generations end at stream version {version} with the amended title": ( + world, + { version }, + ) => { + expect(world.results.length).toBeGreaterThan(0); + expect(world.results.every((value) => value === version)).toBe(true); + }, +}); diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.test.ts new file mode 100644 index 0000000..70cd967 --- /dev/null +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.test.ts @@ -0,0 +1,116 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe6ContextBatchCursorContract as contract } from "../../generated/contracts/facts.f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.contract.js"; +import type { Backend } from "../../harness/backend.js"; +import { + setup, + record, + status, + untilStatus, + completeRows, +} from "./migration-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor", + ), + verifies: ref( + "spec:facts.f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor", + ), +}); +void anchor; +interface World { + backend?: Backend; + batch: number; +} +async function checkpoint(backend: Backend) { + const component = ( + await backend.admin.readTable("migrations", { component: "migrations" }) + ).find((row) => row.name === "migrations:contextBatch")!; + const generation = (await backend.admin.readTable("generations")).find( + (row) => row.generation === 2, + )!; + expect((generation.cursor as { pageCursor: string }).pageCursor).toBe( + component.cursor, + ); + return { component, generation }; +} +bindExample(contract, (): World => ({ batch: 0 }), { + "{rows} depot documents and a parent batch capped at {batch} source rows": + async (world, { rows, batch }) => { + world.backend = (await setup(rows)).backend; + world.batch = batch; + }, + "the migrations driver cancels and resumes the context batch and a later batch throws after writing its checkpoint": + async (world) => { + const backend = world.backend!; + await backend.admin.run("migrations:runContextBatch", { + cancel: true, + batchSize: world.batch, + }); + const canceled = await status( + backend, + "migrations", + "migrations:contextBatch", + ); + expect(canceled.state).toBe("canceled"); + expect(canceled.processed).toBe(world.batch); + const before = await checkpoint(backend); + record("context checkpoint after cancel", { + status: canceled, + ...before, + }); + await backend.admin.run("migrations:configure", { + failKey: "doc-03", + reads: 0, + clear: false, + }); + await backend.admin.run("migrations:runContextBatch", { + batchSize: world.batch, + }); + const failed = await untilStatus( + backend, + "failed", + "migrations", + "migrations:contextBatch", + ); + expect(failed.error).toContain( + "Context batch interrupted after checkpoint write", + ); + const after = await checkpoint(backend); + record("context checkpoint after failure", { status: failed, ...after }); + expect(after.generation).toEqual(before.generation); + expect(after.component.cursor).toBe(before.component.cursor); + expect(await backend.admin.readTable("migrationVisits")).toHaveLength( + world.batch, + ); + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 0, + clear: false, + }); + await backend.admin.run("migrations:runContextBatch", { + batchSize: world.batch, + }); + record("context batch complete", { + status: await untilStatus( + backend, + "success", + "migrations", + "migrations:contextBatch", + ), + ...(await checkpoint(backend)), + }); + }, + "resume writes {rows} distinct summaries and both saved cursors agree at each interrupted boundary": + async (world, { rows }) => { + record("context batch rows", await completeRows(world.backend!, rows, 1)); + expect(await world.backend!.admin.readTable("enumerationPages")).toEqual( + [], + ); + }, +}); diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.test.ts new file mode 100644 index 0000000..be738a9 --- /dev/null +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.test.ts @@ -0,0 +1,180 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe6ContextEnumerationContract as contract } from "../../generated/contracts/facts.f17-migrations-fits-generation-backfill.probe-6-context-enumeration.contract.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f17-migrations-fits-generation-backfill.probe-6-context-enumeration", + ), + verifies: ref( + "spec:facts.f17-migrations-fits-generation-backfill.probe-6-context-enumeration", + ), +}); +void anchor; +import type { Backend } from "../../harness/backend.js"; +import { + setup, + record, + status, + untilStatus, + thrown, +} from "./migration-steps.js"; +interface World { + backend?: Backend; + parentRows?: number; + contextRows?: number; +} +bindExample(contract, (): World => ({}), { + "{rows} depot documents and migrations mounted in the parent and depot": + async (world, { rows }) => { + world.backend = (await setup(rows)).backend; + }, + "parent and depot migrations walk tables and a parent migration calls the context list": + async (world) => { + const backend = world.backend!; + await backend.admin.run("migrations:contextTable", {}); + const parent = await untilStatus( + backend, + "failed", + "migrations", + "migrations:contextTable", + ); + world.parentRows = parent.processed; + record("parent table walk", parent); + expect(parent.error).toContain( + "Uncaught Error: Index streams.by_creation_time not found.", + ); + await backend.admin.run("migrations:contextTableWithoutSchema", {}); + const runtime = await untilStatus( + backend, + "failed", + "migrations", + "migrations:contextTableWithoutSchema", + ); + record("parent walk without schema", runtime); + expect(runtime.processed).toBe(0); + expect(runtime.error).toContain( + "Uncaught Error: Index streams.by_creation_time not found.", + ); + await backend.admin.run("migrations:rangeWithoutSchema", {}); + const missingSchema = await untilStatus( + backend, + "failed", + "migrations", + "migrations:rangeWithoutSchema", + ); + record("custom range without schema", missingSchema); + expect(missingSchema.error).toContain( + "You must provide your schema to use a custom range.", + ); + const source = ( + await backend.admin.readTable("streams", { component: "depot" }) + )[0]!; + const parentIdError = await thrown( + backend.admin.run("migrations:contextId", { id: source._id! }), + ); + record("parent reads context id", parentIdError); + expect(parentIdError).toContain( + 'Invalid argument `id` for `db.get`: expected to be an Id<"streams">, got Id<"depthRows"> instead.', + ); + await backend.admin.run("migrations:streams", {}, { component: "depot" }); + const context = await untilStatus( + backend, + "success", + "depot/migrations", + "migrations:streams", + ); + world.contextRows = context.processed; + record("context table walk", { + status: context, + visits: await backend.admin.readTable("migrationVisits", { + component: "depot", + }), + stored: await backend.admin.readTable("migrations", { + component: "depot/migrations", + }), + }); + const parentRow = ( + await backend.admin.readTable("documentSummaries") + )[0]!; + await backend.admin.run("migrations:prepareContext", { + parentId: parentRow._id!, + }); + await backend.admin.run( + "migrations:parentWrite", + {}, + { component: "depot" }, + ); + const failed = await untilStatus( + backend, + "failed", + "depot/migrations", + "migrations:parentWrite", + ); + record("context writes parent id", failed); + expect(failed.error).toContain( + 'Invalid argument `id` for `db.patch`: expected to be an Id<"documentSummaries">, got Id<"migrationSettings"> instead.', + ); + expect( + (await backend.admin.readTable("documentSummaries"))[0]!.title, + ).toBe("original"); + await backend.admin.run( + "migrations:callback", + {}, + { component: "depot" }, + ); + record( + "context calls parent handle", + await untilStatus( + backend, + "success", + "depot/migrations", + "migrations:callback", + ), + ); + expect( + (await backend.admin.readTable("documentSummaries")).filter( + (row) => row.generation === 2, + ), + ).toHaveLength(world.contextRows!); + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 0, + clear: true, + }); + await backend.admin.writeTable("enumerationPages", { + insert: { tenantId: "tenant", cursor: null, size: 2 }, + }); + await backend.admin.run("migrations:runEnumeration", {}); + record( + "parent enumerates context", + await untilStatus( + backend, + "success", + "migrations", + "migrations:enumeration", + ), + ); + const pages = await backend.admin.readTable("enumerationPages"); + record("enumeration page records", pages); + expect(pages).toHaveLength(3); + expect( + (await backend.admin.readTable("documentSummaries")).filter( + (row) => row.generation === 2, + ), + ).toHaveLength(world.contextRows!); + record( + "context status after enumeration", + await status(backend, "depot/migrations", "migrations:streams"), + ); + }, + "the parent table walk fails with {parentRows} processed context rows and the depot table walk sees {contextRows} rows": + (world, params) => { + expect(world.parentRows).toBe(params.parentRows); + expect(world.contextRows).toBe(params.contextRows); + }, +}); diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.test.ts new file mode 100644 index 0000000..4e7c987 --- /dev/null +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.test.ts @@ -0,0 +1,131 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe6InterruptionAndResumeContract as contract } from "../../generated/contracts/facts.f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume.contract.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume", + ), + verifies: ref( + "spec:facts.f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume", + ), +}); +void anchor; +import type { Backend } from "../../harness/backend.js"; +import { + setup, + record, + status, + untilStatus, + completeRows, +} from "./migration-steps.js"; +interface World { + backend?: Backend; + rows: number; + batch: number; +} +bindExample(contract, (): World => ({ rows: 0, batch: 0 }), { + "{rows} document summaries and a migration with batch size {batch}": async ( + world, + { rows, batch }, + ) => { + world.backend = (await setup(rows)).backend; + world.rows = rows; + world.batch = batch; + }, + "the migration is canceled, resumed, failed inside a batch and resumed, then the backend restarts with a batch pending": + async (world) => { + const backend = world.backend!; + const canceled = await backend.admin.run("migrations:run", { + cancel: true, + batchSize: world.batch, + }); + const state = await status(backend); + record("cancel", { + returned: canceled, + status: state, + stored: await backend.admin.readTable("migrations", { + component: "migrations", + }), + }); + expect(state.state).toBe("canceled"); + expect(state.processed).toBe(world.batch); + expect(state.cursor).not.toBeNull(); + expect(await backend.admin.readTable("migrationVisits")).toHaveLength( + world.batch, + ); + await backend.admin.run("migrations:run", { batchSize: world.batch }); + record("resumed", await untilStatus(backend, "success")); + await completeRows(backend, world.rows, 1); + await backend.admin.run("migrations:configure", { + failKey: "doc-03", + reads: 0, + clear: true, + }); + await backend.admin.run("migrations:run", { + reset: true, + cancel: true, + batchSize: world.batch, + }); + const before = await status(backend); + await backend.admin.run("migrations:run", { batchSize: world.batch }); + const failed = await untilStatus(backend, "failed"); + record("thrown batch", { + before, + failed, + visits: await backend.admin.readTable("migrationVisits"), + stored: await backend.admin.readTable("migrations", { + component: "migrations", + }), + }); + expect(failed.error).toContain( + "Backfill batch interrupted after row write", + ); + expect(failed.cursor).toBe(before.cursor); + expect(failed.processed).toBe(before.processed); + expect(await backend.admin.readTable("migrationVisits")).toHaveLength( + world.batch, + ); + expect( + (await backend.admin.readTable("documentSummaries")).filter( + (row) => row.generation === 2, + ), + ).toHaveLength(world.batch); + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 0, + clear: false, + }); + await backend.admin.run("migrations:run", { batchSize: world.batch }); + record("resumed after throw", await untilStatus(backend, "success")); + await completeRows(backend, world.rows, 1); + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 200, + clear: true, + }); + await backend.admin.run("migrations:run", { reset: true, batchSize: 1 }); + const scheduled = await backend.admin.readTable("_scheduled_functions", { + component: "migrations", + }); + const pending = scheduled.filter( + (row) => (row.state as { kind: string }).kind === "pending", + ); + record("before restart", { status: await status(backend), pending }); + expect(pending.length).toBeGreaterThan(0); + await backend.kill(); + await backend.restart(); + record("after restart", await untilStatus(backend, "success")); + }, + "every source has exactly {copies} target row and a successful visit after each completed migration": + async (world, { copies }) => { + record( + "final rows", + await completeRows(world.backend!, world.rows, copies), + ); + }, +}); diff --git a/tests/native/migration-steps.ts b/tests/native/migration-steps.ts new file mode 100644 index 0000000..2523e22 --- /dev/null +++ b/tests/native/migration-steps.ts @@ -0,0 +1,117 @@ +import { setTimeout as sleep } from "node:timers/promises"; +import { expect } from "vitest"; +import type { MigrationStatus } from "@convex-dev/migrations"; +import type { Value } from "convex/values"; +import { api } from "../../fixture/convex/_generated/api.js"; +import type { Backend } from "../../harness/backend.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend, measure } from "../../harness/native.js"; +import type { JsonValue } from "../../harness/evidence.js"; +export const record = (name: string, value: unknown) => + measure(name, JSON.parse(JSON.stringify(value)) as JsonValue); +export async function setup(rows: number) { + const backend = await fixtureBackend(); + record("pins", { + backend: "precompiled-2026-09-28-5c7cb5b", + convex: "1.46.0", + "convex-helpers": "0.1.124", + "@convex-dev/migrations": "0.3.6", + }); + await backend.admin.run("readModels:activate", { + readModel: "documentSummary", + startedBy: { kind: "operator", id: "native-test" }, + }); + const generation = (await backend.admin.readTable("generations"))[0]!; + const { _id, _creationTime, switchedAt, ...fields } = generation; + void _id; + void _creationTime; + void switchedAt; + await backend.admin.writeTable("generations", { + insert: { ...fields, generation: 2, state: "building", batchSize: 2 }, + }); + await backend.admin.run("grants:grant", { + tenantId: "tenant", + principalKind: "human", + principalId: `${backend.issuer.issuer}|writer`, + permission: "depot.documents", + grantedBy: "native-test", + }); + const client = ordinaryClient(backend.url, { + token: await backend.issuer.token("writer"), + }); + for (let i = 0; i < rows; i++) + await client.mutation(api.depotCommands.createSummarizedDocument, { + tenantId: "tenant", + input: { + documentId: `doc-${String(i).padStart(2, "0")}`, + title: "original", + }, + }); + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 0, + clear: true, + }); + return { backend, client }; +} +export async function status( + backend: Backend, + component = "migrations", + name = "migrations:summaries", +) { + const result = await backend.admin.run( + "lib:getStatus", + { names: [name] }, + { component }, + ); + return (result as unknown as MigrationStatus[])[0]!; +} +export async function untilStatus( + backend: Backend, + state: string, + component = "migrations", + name = "migrations:summaries", +) { + const deadline = Date.now() + 25000; + while (Date.now() < deadline) { + const result = await status(backend, component, name); + if (result.state === state) return result; + if (result.state === "failed" && state !== "failed") + throw new Error(JSON.stringify(result)); + await sleep(20); + } + throw new Error( + `Migration did not reach ${state}: ${JSON.stringify(await status(backend, component, name))}`, + ); +} +export async function completeRows( + backend: Backend, + rows: number, + copies: number, +) { + const target = (await backend.admin.readTable("documentSummaries")).filter( + (row) => row.generation === 2, + ); + const visits = await backend.admin.readTable("migrationVisits"); + expect(target).toHaveLength(rows); + expect(target.map((row) => row.key).sort()).toEqual( + Array.from( + { length: rows }, + (_, index) => `doc-${String(index).padStart(2, "0")}`, + ), + ); + expect(visits).toHaveLength(rows * copies); + for (const row of target) + expect(visits.filter((visit) => visit.key === row.key)).toHaveLength( + copies, + ); + return { target, visits }; +} +export async function thrown(promise: Promise) { + return promise.then( + () => { + throw new Error("The boundary did not refuse the call"); + }, + (error) => String(error), + ); +} From 032bff7fa75f2a9036549091c19cc4e4e8db51f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:17:37 +0200 Subject: [PATCH 015/155] test: read of the measurement: conflict tables with their component, the engine's retry count, the deployed tables checked, the record type spelled out The table a conflict named now carries the component that holds it, and each request's refused executions are checked against the engine's own retry count. Rows left behind are reported as not deployed only while the composition's schemas declare no such table. The Measurement record type in the first-experiment Spec names the shapes of the latency and throughput readings, the paths and the caller answers the code writes. The unrelated-stock tests say which tenant layout they run, and the timing command keeps a backend binary the caller exported. Co-Authored-By: Claude Opus 5.5 --- .../specs/application/first-experiment.sdp.md | 10 ++-- scripts/measure-place-order.mjs | 3 +- tests/native/place-order-measurement.test.ts | 9 ++-- tests/native/place-order-measurement.ts | 49 +++++++++++++------ 4 files changed, 48 insertions(+), 23 deletions(-) diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index ef1fd98..8d3397f 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -98,16 +98,18 @@ The six cost targets are the constraint Specs this workflow is constrained by. T The experiment is an application, a test suite and a report. The application is the production composition: two context components, the parent with its pipeline, use cases, read models, registry and gate. The suite is the acceptance scenarios of Layers 0 to 2, run against a disposable local backend as the acceptance contract prescribes. The report is a set of measurement records, one per run, compared against the six constraint targets. The production composition is the Convex app whose project directory is `example/`: its functions are in `example/convex/`, its two contexts are the components `example/convex/orders/` and `example/convex/inventory/`, and its deciders are in `example/domain/`. -- typeMeasurement: [extension] `type Measurement = { recordType: "Measurement"; commit: string; pins: MeasurementPins; configuration: "production" | "adjusted"; adjustments?: string; identitySource: "fixture-issuer" | "production-issuer"; installedLayers: string[]; dataset: string; command: "PlaceOrder" | "CancelOrder"; lines: number; contention: number; path: string; submittedCommands: number; executions: number; finalOutcomes: number; outcomeCounts: Record; topLevelCommits: number; functionCalls: number; componentCalls: number | "not exposed"; rolledBackWrites: number | "not exposed"; writeSetFromLog: "not exposed"; usageStats: MeasurementUsage; finalUsageStats: MeasurementUsage; rerunUsageStats: MeasurementUsage; engineReruns: number; engineFailedCommands: number; occTables: string[]; rowsBefore: Record; rowsAfter: Record; rowsLeftBehind: Record; requests: MeasurementRequest[]; callerAnswers: unknown[]; requestToInput: string; writeCapBytesPerSecond: number; timingReading: "taken under load" | "quiet machine"; latencyDistribution: unknown; throughput: unknown }`; the native helper names this record `PlaceOrderMeasurement`, apart from the harness name/value envelope named `Measurement` (E-47, E-13, First experiment) +- typeMeasurement: [extension] `type Measurement = { recordType: "Measurement"; commit: string; pins: MeasurementPins; configuration: "production" | "adjusted"; adjustments?: string; identitySource: "fixture-issuer" | "production-issuer"; installedLayers: string[]; dataset: string; command: "PlaceOrder" | "CancelOrder"; lines: number; contention: number; path: "healthy" | "authorized duplicate" | "changed input conflict" | "business rejection" | "second-context failure" | "current state differs from fold" | "unrelated stock" | "contention" | "latency" | "throughput"; submittedCommands: number; executions: number; finalOutcomes: number; outcomeCounts: Record; topLevelCommits: number; functionCalls: number; componentCalls: number | "not exposed"; rolledBackWrites: number | "not exposed"; writeSetFromLog: "not exposed"; usageStats: MeasurementUsage; finalUsageStats: MeasurementUsage; rerunUsageStats: MeasurementUsage; engineReruns: number; engineFailedCommands: number; occTables: string[]; rowsBefore: Record; rowsAfter: Record; rowsLeftBehind: Record; requests: MeasurementRequest[]; callerAnswers: { kind: string; code: string | null; replayed: boolean }[]; requestToInput: string; writeCapBytesPerSecond: number; timingReading: "taken under load" | "quiet machine"; latencyDistribution: MeasurementLatency | null; throughput: MeasurementThroughput | null }`; a cell adds under keys of its own what only it observes, such as the scheduled functions per component or the discarded completions of a latency cell; the native helper names this record `PlaceOrderMeasurement`, apart from the harness name/value envelope named `Measurement` (E-47, E-13, First experiment) - typeMeasurementPins: [extension] `type MeasurementPins = { tier: "native backend"; environment: "local backend"; backend: { release: string; sha256: string; source: string }; convex: string; convexHelpers: string; convexTest: string; node: string; operatingSystem: string }` records backend, dependencies, Node and operating system in every cell; no local number claims a hosted result (E-47, E-15, Sc L2-9) - typeMeasurementUsage: [extension] `type MeasurementUsage = { databaseReadDocuments: number; databaseWriteDocuments: number; databaseReadBytes: number; databaseWriteBytes: number; databaseWriteIndexRows: number }` carries only counters the native backend recorded (E-47, E-15) -- typeMeasurementRequest: [extension] `type MeasurementRequest = { requestId: string; executions: number; finalOutcomes: number; finalError: string | null; engineReruns: number; occTables: string[]; usageStats: MeasurementUsage; finalUsageStats: MeasurementUsage; rerunUsageStats: MeasurementUsage; latencyMs: number; completionRecords: unknown[] }` keeps every completion in its backend request; concurrent HTTP answers are retained separately because the response exposes no backend request ID, so their order never supplies a guessed correspondence (E-47, E-15) +- typeMeasurementRequest: [extension] `type MeasurementRequest = { requestId: string; executions: number; finalOutcomes: number; finalError: string | null; engineReruns: number; occTables: string[]; usageStats: MeasurementUsage; finalUsageStats: MeasurementUsage; rerunUsageStats: MeasurementUsage; latencyMs: number; completionRecords: CompletionRecord[] }` keeps every completion in its backend request; concurrent HTTP answers are retained separately because the response exposes no backend request ID, so their order never supplies a guessed correspondence (E-47, E-15) +- typeMeasurementLatency: [extension] `type MeasurementLatency = { acceptedCommands: number; discardedCommands: number; medianMs: number; p95Ms: number; medianTargetMs: number; p95TargetMs: number; median: "held" | "not held"; p95: "held" | "not held"; percentileMethod: string; executionTimesMs: number[] }` holds every accepted execution time beside the targets it is compared with (E-47, First experiment) +- typeMeasurementThroughput: [extension] `type MeasurementThroughput = { callers: number; lines: number; elapsedSeconds: number; completedOrders: number; ordersPerSecond: number; targetOrdersPerSecond: number; target: "held" | "not held"; writeCapBytesPerSecond: number }` states the local write cap beside every throughput (E-47, First experiment) - captureTiming: [extension] latency is the backend log's top-level `executionTime`, converted to milliseconds, over thirty accepted commands per size after five discarded; the median averages the middle pair and the 95th percentile uses nearest rank; throughput is applied ten-line orders divided by elapsed wall time over at least ten seconds with eight callers on mutually disjoint stock, with the local write cap of 4194304 bytes per second recorded beside it; poll the function log throughout the throughput cell and fail if any poll may have lost entries or the count of terminal records differs from submitted commands (E-47, F13) - captureCommits: top-level commits are terminal top-level mutation completion records with `willRetry` false and `error` null; a completion with `willRetry` true is no commit even when `error` is null, and an authorized duplicate may commit a read-only transaction with no new business effect (E-47, First experiment) - captureCalls: function calls count the command's top-level completion records, including executions the engine reran; component calls and their individual time are `not exposed` by the native backend log, and a separate pure test of the use case executor counts calls by context with canned outcomes at each size (E-47, F4, E-15) - captureDocuments: documents and bytes come only from each top-level completion record's `usageStats`; the record retains every execution and sums its recorded counters by backend request ID and by cell, separately for terminal executions and executions refused at commit, without deriving rolled-back writes or a write set from stored rows; a field the backend does not expose is `not exposed` (E-47, First experiment, E-15) - captureRowsLeftBehind: rows left behind are the documents a run creates that are not state, events, receipts or read-model rows; record their before, after and created counts by table, separately from setup, including markers, registry, gate and audit rows, and say `not deployed` for a table absent from the deployed schema (E-47, First experiment, D19) -- captureRetries: one completion with `willRetry` true and an `occInfo` is one execution refused at commit; its rerun has another completion with the same backend request ID; record these executions, the tables named by `occInfo`, each command's terminal record and final outcomes separately, and count terminal optimistic-concurrency failures as commands the engine failed; the variable number of reruns is an observation, never the difference between two cells (E-47, F1) +- captureRetries: one completion with `willRetry` true and an `occInfo` is one execution refused at commit; its rerun has another completion with the same backend request ID; record these executions, the tables named by `occInfo` with the component that holds each, the engine's own `retryCount` of each refused execution, which numbers them from zero, each command's terminal record and final outcomes separately, and count terminal optimistic-concurrency failures as commands the engine failed; the variable number of reruns is an observation, never the difference between two cells (E-47, F1) - datasetSizes: 1 line, 10 lines and 100 lines as the provisional maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) - contentionShape: at each size, send N `PlaceOrder` commands concurrently at N of 2, 8 and 32, sharing only one stock item with quantity one; state the placement of that item and the layout of the other lines in the dataset; at N of 2 and 8 require one applied outcome, N minus one short-stock rejections and zero commands the engine failed, while at N of 32 report those counts and engine failures without treating an engine failure as an experiment failure; no caller retries during the cell, and the caller may repeat a failed command under its same request key (First experiment, Sc L1-11, F1, E-5, E-47) - limitOrderLines: the maximum number of lines per `PlaceOrder` is a product decision, open under OQ3; the experiment measures at 100 and reports documents read and written per line, so the owner can set it under the 1 second timeout and the F13 ceilings (OQ3, F13) @@ -115,7 +117,7 @@ The experiment is an application, a test suite and a report. The application is - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate read, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer (E-47, F13) - observedMeasurementPins: [extension] the measurement records name the native backend tier, the local backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124, `convex-test` 0.0.60, Node v24.19.0 and Linux 7.2.7-200.fc44.x86_64 on x64; these observations concern that local environment only (E-47, E-15, Sc L2-9) - observedMeasurementBytes: [extension] at the native backend tier with `observedMeasurementPins`, fixed seven-character stock and order IDs, four grants and one active generation, orders of 1, 10 and 100 distinct stock items received once read 2943, 14454 and 129564 bytes and write 4271, 18410 and 159800 bytes; the corresponding write-index rows are 25, 97 and 817; a ten-line authorized duplicate or changed-input conflict reads 5 documents and 3441 bytes and writes zero, while short stock or a missing journal tail on the last line reads 32 documents and 13106 bytes and writes zero (E-47, F13) -- observedContentionCosts: [extension] at the native backend tier with `observedMeasurementPins`, each of the nine cells at 1, 10 and 100 lines and 2, 8 and 32 callers reached one applied outcome and all remaining short-stock rejections with zero engine failures; an execution refused at commit read the healthy path's documents and bytes and wrote zero, and its final short-stock rejection, with the shared item first, read 5 documents and 1595 bytes and wrote zero; `occInfo` named Inventory's events or streams across the cells, and the rerun count remains an observation per record (E-47, F1) +- observedContentionCosts: [extension] at the native backend tier with `observedMeasurementPins`, each of the nine cells at 1, 10 and 100 lines and 2, 8 and 32 callers reached one applied outcome and all remaining short-stock rejections with zero engine failures; an execution refused at commit read the healthy path's documents and bytes and wrote zero, and its final short-stock rejection, with the shared item first, read 5 documents and 1595 bytes and wrote zero; `occInfo` named Inventory's events or streams table across the cells and its `retryCount` was zero, one refused execution per rejected command, and the rerun count remains an observation per record (E-47, F1) - observedUnrelatedStock: [extension] at the native backend tier with `observedMeasurementPins`, eight simultaneous ten-line commands on disjoint stock in one tenant and in eight unrelated tenants each recorded 8 executions, 8 applied outcomes and zero engine reruns; each command read 35 documents and 14454 bytes and wrote 24 documents and 18410 bytes, and the shared generation row was unchanged (E-47, F1) - observedStockFoldGap: [extension] at the native backend tier with `observedMeasurementPins`, after `ReceiveStock` recorded quantity two, an admin change of current on-hand stock to three with the same stream version and journal tail let a one-line order allocate three, reading 8 documents and 2943 bytes and writing 6 documents and 4271 bytes; matching the tail version does not establish equality of current state with the complete event fold (E-47, D3) - streamBudgets: the stock item stream declares `budgetBytes` of 16 KiB and the order stream keeps the default 256 KiB, as `spec:application.orders-inventory-example` pins, so the allocation call's byte bound, 8 MiB over the planned streams' budgets, admits 512 stock streams and the adapter's count bound of `min(maxStreams, 256)` admits the 100 of the maximum order with room, while the single order stream costs at most 256 KiB; at the default budget on both stream types the byte bound would admit only 32 stock streams and the maximum run would be rejected `operationTooLarge`, which is why the budget is pinned here and the run at the maximum measures it (E-2, E-22, E-46, F13, Sc L2-3) diff --git a/scripts/measure-place-order.mjs b/scripts/measure-place-order.mjs index 43f020d..fd9bb17 100644 --- a/scripts/measure-place-order.mjs +++ b/scripts/measure-place-order.mjs @@ -12,7 +12,8 @@ environment.PLACE_ORDER_TIMINGS = "1"; environment.PLACE_ORDER_TIMING_LOAD = process.argv.includes("--quiet") ? "quiet" : "under load"; -environment.CONVEX_BACKEND_BINARY = join( +// The pinned backend binary of the owner's machine, unless the caller exported another path. +environment.CONVEX_BACKEND_BINARY ??= join( homedir(), "dev-libar/libar-platform/.cache/precompiled-2026-09-28-5c7cb5b/convex-local-backend-x86_64-unknown-linux-gnu.zip/convex-local-backend", ); diff --git a/tests/native/place-order-measurement.test.ts b/tests/native/place-order-measurement.test.ts index a368060..fd45e05 100644 --- a/tests/native/place-order-measurement.test.ts +++ b/tests/native/place-order-measurement.test.ts @@ -164,9 +164,12 @@ test.each([ } }); -test.each([false, true])( - "native: unrelated stock commands with separate tenants %s", - async (separateTenants) => { +test.each([ + ["in one tenant", false], + ["in eight tenants", true], +] as const)( + "native: eight PlaceOrder commands at once on unrelated stock %s", + async (_tenants, separateTenants) => { const world = await setup(); const args = Array.from({ length: 8 }, (_, i) => order(10, i, separateTenants ? `t-${i + 1}` : "t-1"), diff --git a/tests/native/place-order-measurement.ts b/tests/native/place-order-measurement.ts index 4c366f6..a88099c 100644 --- a/tests/native/place-order-measurement.ts +++ b/tests/native/place-order-measurement.ts @@ -5,6 +5,8 @@ import { getFunctionName, type FunctionReturnType } from "convex/server"; import { ConvexError } from "convex/values"; import { expect } from "vitest"; import { api, internal } from "../../example/convex/_generated/api.js"; +import schema from "../../example/convex/schema.js"; +import { contextTables } from "../../src/context/index.js"; import type { CompletionRecord, LogMark } from "../../harness/admin.js"; import type { Backend } from "../../harness/backend.js"; import type { JsonValue } from "../../harness/evidence.js"; @@ -32,8 +34,21 @@ import { export type ExecutionRecord = CompletionRecord & { willRetry: boolean; - occInfo: { tableName: string } | null; + occInfo: { + tableName: string; + componentPath: string | null; + retryCount: number; + } | null; }; +// The table an optimistic-concurrency conflict named, with the component that holds it. +const occTable = (record: ExecutionRecord) => + record.occInfo === null + ? [] + : [ + record.occInfo.componentPath === null + ? record.occInfo.tableName + : `${record.occInfo.componentPath}:${record.occInfo.tableName}`, + ]; export type Answer = { kind: string; code: string | null; @@ -186,7 +201,16 @@ export async function schedules(backend: Backend) { ); return Object.fromEntries(entries); } +// Rows left behind are counted over every table the production composition deploys; markers, the +// gate and audit records are reported as not deployed only while neither schema declares a table. export function leftBehind(before: Stored, after: Stored) { + expect(Object.keys(schema.tables).sort()).toEqual([ + "generations", + "grants", + "orderSummaries", + "receipts", + ]); + expect(Object.keys(contextTables).sort()).toEqual(["events", "streams"]); return { generations: { before: before.generations.length, @@ -232,13 +256,7 @@ export function recordMeasurement( finalOutcomes: 1, finalError: final.error, engineReruns: reruns.length, - occTables: [ - ...new Set( - executions.flatMap((record) => - record.occInfo === null ? [] : [record.occInfo.tableName], - ), - ), - ], + occTables: [...new Set(executions.flatMap(occTable))], usageStats: sumUsage(executions), finalUsageStats: usageOf(final), rerunUsageStats: sumUsage(reruns), @@ -297,13 +315,7 @@ export function recordMeasurement( engineReruns: own.filter((record) => record.willRetry).length, engineFailedCommands: completed.filter((record) => record.occInfo !== null) .length, - occTables: [ - ...new Set( - own.flatMap((record) => - record.occInfo === null ? [] : [record.occInfo.tableName], - ), - ), - ], + occTables: [...new Set(own.flatMap(occTable))], rowsBefore: rowCounts(options.before), rowsAfter: rowCounts(options.after), rowsLeftBehind: leftBehind(options.before, options.after), @@ -328,6 +340,13 @@ export function recordMeasurement( expect( request.completionRecords.filter((record) => !record.willRetry), ).toHaveLength(1); + // The engine numbers the executions it refused at commit, so its own count agrees with ours. + expect( + request.completionRecords + .filter((record) => record.willRetry) + .map((record) => record.occInfo?.retryCount), + ).toEqual(request.completionRecords.slice(0, -1).map((_, index) => index)); + expect(request.completionRecords.at(-1)?.willRetry).toBe(false); for (const record of request.completionRecords) { expect(typeof record.willRetry).toBe("boolean"); expect(record).toHaveProperty("occInfo"); From 55c7643f103255460c7e719c74ce8dc2843daaa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:22:02 +0200 Subject: [PATCH 016/155] test: keep the log window beside the scheduled-function read; the fact Specs in the corpus form The no-job assertions of the order summary and the three order sizes keep their function-log window and add the read of _scheduled_functions in the parent and both contexts, taken once after the commands, so that place() makes no extra read. Each of the five example Specs changes only its clause about a delayed job. F18 to F20 and their examples take the form of the earlier facts and probes: the status line, a narrative paragraph, an outcome per fact and per example, and step texts that read as sentences. F19 states the range of times seen under load and asserts none. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 14 ++++----- .../first-experiment.order-of-1-line.sdp.md | 2 +- .../first-experiment.order-of-10-lines.sdp.md | 2 +- ...first-experiment.order-of-max-lines.sdp.md | 2 +- ....committed-state-read-through-query.sdp.md | 2 +- ...read-models.committed-state-visible.sdp.md | 2 +- ...tamps.probe-8-parent-and-components.sdp.md | 13 +++++---- .../specs/facts/f18-commit-timestamps.sdp.md | 24 ++++++++------- ...me-budgets.probe-11-component-calls.sdp.md | 14 ++++----- ...re-time-budgets.probe-9-nested-time.sdp.md | 16 +++++----- ...f19-nested-calls-share-time-budgets.sdp.md | 24 ++++++++------- ...fter-reading.probe-10-oversized-row.sdp.md | 12 ++++---- ...fter-reading.probe-10-stream-budget.sdp.md | 13 +++++---- ...gination-checks-bytes-after-reading.sdp.md | 18 +++++++----- design/specs/facts/fact-ledger.sdp.md | 2 +- design/specs/facts/probe-plan.sdp.md | 11 +++---- ...e-budgets.probe-11-component-calls.test.ts | 2 +- ...e-time-budgets.probe-9-nested-time.test.ts | 8 +++-- ...ter-reading.probe-10-oversized-row.test.ts | 2 +- ...ter-reading.probe-10-stream-budget.test.ts | 2 +- tests/native/first-experiment-steps.ts | 29 ++++++++++++------- ...committed-state-read-through-query.test.ts | 18 ++++++++++-- ...ead-models.committed-state-visible.test.ts | 18 ++++++++++-- 23 files changed, 149 insertions(+), 101 deletions(-) diff --git a/design/README.md b/design/README.md index e7b561d..7ea8989 100644 --- a/design/README.md +++ b/design/README.md @@ -276,8 +276,8 @@ Probes 1 to 5 ran on a native backend in slice S0 on 2026-10-01. Each result is | Tier | Fact | Subject | |---|---|---| -| Native backend | [`spec:facts.f18-commit-timestamps`](specs/facts/f18-commit-timestamps.sdp.md) | Commit timestamps order committed rows | -| Native backend | [`spec:facts.f19-nested-calls-share-time-budgets`](specs/facts/f19-nested-calls-share-time-budgets.sdp.md) | Nested calls share time budgets | +| Native backend | [`spec:facts.f18-commit-timestamps`](specs/facts/f18-commit-timestamps.sdp.md) | Commit timestamps order committed transactions | +| Native backend | [`spec:facts.f19-nested-calls-share-time-budgets`](specs/facts/f19-nested-calls-share-time-budgets.sdp.md) | Nested calls draw on a system-operation time budget | | Native backend | [`spec:facts.f20-pagination-checks-bytes-after-reading`](specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md) | Pagination checks bytes after reading | Every open question in the corpus, by Spec, is recipe 20 of the Protocol's catalog, the open-question register. It lists the Specs that hold a blocking question first and reports the totals, so no table is kept here: @@ -324,11 +324,11 @@ The Layer 2 design added two more for the same reason, and nine examples that ve | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split` | Probe 5: a full first page under a row cap of twice the page size is not split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap` | Probe 5: the same page under a row cap equal to the page size is split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap` | Probe 5: a parent list's page outgrows its row cap | -| none | [`spec:facts.f18-commit-timestamps.probe-8-parent-and-components`](specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md) | Commit timestamps in the parent and two components | -| none | [`spec:facts.f19-nested-calls-share-time-budgets.probe-11-component-calls`](specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md) | Component calls per mutation | -| none | [`spec:facts.f19-nested-calls-share-time-budgets.probe-9-nested-time`](specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md) | Nested calls reach a time boundary | -| none | [`spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-oversized-row`](specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md) | A page keeps the row that crosses its byte bound | -| none | [`spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-stream-budget`](specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md) | A page includes stored system fields in its byte check | +| none | [`spec:facts.f18-commit-timestamps.probe-8-parent-and-components`](specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md) | Probe 8: commit timestamps in the parent and two components | +| none | [`spec:facts.f19-nested-calls-share-time-budgets.probe-11-component-calls`](specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md) | Probe 11: component calls per mutation | +| none | [`spec:facts.f19-nested-calls-share-time-budgets.probe-9-nested-time`](specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md) | Probe 9: nested calls reach a time boundary | +| none | [`spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-oversized-row`](specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md) | Probe 10: a page keeps the row that crosses its byte bound | +| none | [`spec:facts.f20-pagination-checks-bytes-after-reading.probe-10-stream-budget`](specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md) | Probe 10: a page of full stream rows reads past the library's byte bound | Package A therefore has 87 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. diff --git a/design/specs/application/first-experiment.order-of-1-line.sdp.md b/design/specs/application/first-experiment.order-of-1-line.sdp.md index baef41d..7ff43d5 100644 --- a/design/specs/application/first-experiment.order-of-1-line.sdp.md +++ b/design/specs/application/first-experiment.order-of-1-line.sdp.md @@ -29,6 +29,6 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends `PlaceOrder` with one line, reads the backend's function log from a mark before the command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for the command, which committed, and reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, asserting that all three tables are empty, including any function scheduled with a delay, and that the list the read returned already holds the order's summary row. +- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends `PlaceOrder` with one line, reads the backend's function log from a mark before the command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for the command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation for this size. - The test reads the documents and bytes read and written from the completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts six documents written: the receipt, the order's state and event, the stock item's state and event, and the summary row; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-10-lines.sdp.md b/design/specs/application/first-experiment.order-of-10-lines.sdp.md index afeb002..51c5502 100644 --- a/design/specs/application/first-experiment.order-of-10-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-10-lines.sdp.md @@ -29,6 +29,6 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` with ten lines over ten stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, asserting that all three tables are empty, including any function scheduled with a delay, and that the list the read returned already holds the order's summary row. +- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` with ten lines over ten stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation carrying the order and then one on the Inventory operation carrying all ten lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts that the ten-line order wrote two documents more per added line than the one-line reference, the added stock item's state and its event, so documents written grew linearly with the lines; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-max-lines.sdp.md b/design/specs/application/first-experiment.order-of-max-lines.sdp.md index 3890b6a..2a56a23 100644 --- a/design/specs/application/first-experiment.order-of-max-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-max-lines.sdp.md @@ -29,7 +29,7 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the provisional maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, asserting that all three tables are empty, including any function scheduled with a delay, and that the list the read returned already holds the order's summary row; and it asserts that the completion record's execution time is under the 1 second mutation timeout. +- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the provisional maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row; and it asserts that the completion record's execution time is under the 1 second mutation timeout. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation carrying all 100 lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, and asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. - The test then receives stock for 101 more stock items through `ReceiveStock`, sends `PlaceOrder` with one line on each, one line more than the maximum, and asserts the `operationTooLarge` rejection whose details carry 101 items against a maximum of 100, with the receipts, both contexts' stream rows and events and the order summaries unchanged. diff --git a/design/specs/application/read-models.committed-state-read-through-query.sdp.md b/design/specs/application/read-models.committed-state-read-through-query.sdp.md index 415cf03..162c977 100644 --- a/design/specs/application/read-models.committed-state-read-through-query.sdp.md +++ b/design/specs/application/read-models.committed-state-read-through-query.sdp.md @@ -26,4 +26,4 @@ And the number of workers, jobs or queues that ran is {workers: 0} - Runs in the native tier on the production composition; every test owns its disposable backend. - After grants, the first activation and `ReceiveStock` as setup, the test runs `PlaceOrder` and then, from the same client, calls the parent query `getOrder`, which relays the Orders context's `get`, and asserts that the DTO it returns holds the order's committed state and that its `version` equals the command's returned `versions` entry for the order's stream. -- The test reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, and asserts that all three tables are empty, including any function scheduled with a delay. +- The test asserts that no scheduled function, cron or action ran for the command by reading the disposable backend's function log from a mark before the command to the `getOrder` read that the same client sends after the command returned, a window an ordinary read closes, and finding no record that a client did not cause; and it asserts that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either. diff --git a/design/specs/application/read-models.committed-state-visible.sdp.md b/design/specs/application/read-models.committed-state-visible.sdp.md index 45154b5..1e2dd0c 100644 --- a/design/specs/application/read-models.committed-state-visible.sdp.md +++ b/design/specs/application/read-models.committed-state-visible.sdp.md @@ -27,4 +27,4 @@ And the number of workers, jobs or queues that ran is {workers: 0} - Runs in the native tier on the production composition; every test owns its disposable backend. - After grants, the first activation and `ReceiveStock` as setup, the test subscribes to `listOrderSummaries` with the tenant, the status a placed order has and `{ cursor: null, numItems: 10 }`, runs `PlaceOrder`, and asserts that the next subscription value's page carries the order's summary whose `sourceVersions` equal the command's returned `versions` entry for the order's stream, the one stream the summary is projected from. -- The test reads `_scheduled_functions` through admin access after the command in the parent and both mounted components, Orders and Inventory, and asserts that all three tables are empty, including any function scheduled with a delay. +- The test asserts that the backend ran no scheduled function, cron or action between the command and the subscription update, by reading the disposable backend's function log from before the command until it holds the command's mutation and the subscription's query, a window an ordinary read after the command closes, and finding no record that a client did not cause; and it asserts that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either. diff --git a/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md b/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md index 491c3fa..0919ff8 100644 --- a/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md +++ b/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md @@ -7,13 +7,13 @@ relations: refines: spec:facts.f18-commit-timestamps verifies: spec:facts.f18-commit-timestamps --- -# Commit timestamps in the parent and two components +# Probe 8: commit timestamps in the parent and two components -Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. +Probe 8 · native tier · fixture composition. ## Intent -- outcome: Establish the observed behavior on the pinned local backend. +- outcome: One transaction writes one commit timestamp in the parent and in two components, later transactions write larger ones, and the placeholder resolves when a mutation returns it and is refused as a scheduled argument. (Probe 8, F18) ```gwt Given a parent and two components that write commit timestamp placeholders @@ -24,7 +24,8 @@ And returning a placeholder {returned: "resolves to a bigint"} and scheduling it ## Verification — executable -- Every test owns a disposable native backend. Timings and call counts are recorded, not asserted, and say nothing about a hosted deployment. -- The first native run refuted the expectation that returning a mutation placeholder is refused. The returned value is the committed bigint, which this example binds as observed; the scheduling refusal held. -- The query uses the internal runtime `getSnapshotTs()` through an explicit cast. A compiled type assertion checks that the public `QueryMeta` declaration has no such method. This example does not establish a supported public upper bound or hosted behavior. +- Runs in the native tier on the fixture composition, where `annex` is mounted twice so that two components write; every test owns its disposable backend. +- Inside the writing mutation the test asserts that a read of the row yields the unresolved placeholder, in the parent and in each component, and in each component that converting it to a number throws. +- The expectation written before the first run was that returning the placeholder is refused; the pinned release returned the committed bigint, and the example is bound to that observed value. The refusal to schedule it held, with `Field name $commitTs starts with '$', which is reserved.` +- The query uses the internal runtime `getSnapshotTs()` through an explicit cast. A compiled type assertion checks that the public `QueryMeta` declaration has no such method. A local backend cannot say whether a hosted deployment behaves the same, and no public upper bound is shown; both stay open on F18. - The returned bigint equals the stored row timestamp and exceeds the earlier query upper bound; the failed scheduled call leaves no row and no scheduled function. diff --git a/design/specs/facts/f18-commit-timestamps.sdp.md b/design/specs/facts/f18-commit-timestamps.sdp.md index 158fdcd..9d3699c 100644 --- a/design/specs/facts/f18-commit-timestamps.sdp.md +++ b/design/specs/facts/f18-commit-timestamps.sdp.md @@ -6,29 +6,31 @@ readiness: defined relations: refines: spec:facts.fact-ledger --- -# Commit timestamps order committed rows +# Commit timestamps order committed transactions -F18 · Native backend tier · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. +F18 · Status: probed · Doc status: not in the doc's ledger · Decisions: D2, D18. + +The pinned package declares `db.vars.commitTs`, a placeholder that a mutation writes into a field and that resolves at commit to a bigint ordered by commit order. Probe 8 ran it on the native backend, release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0: one transaction gets one timestamp in the parent and in two components, and later transactions get larger ones. D2 cites the doc's sentence that no deployment-wide order token exists, and whether this fact changes that sentence is the owner's. A timestamp is shared by every row one transaction writes, so it orders transactions and not the events inside one. A reader's upper bound, `getSnapshotTs()`, is marked internal in the package source and absent from its public declarations, so it is no public promise of the pinned release. ## Intent -- outcome: Record the package declaration and native backend behavior with the limits of the evidence. (F18) +- outcome: Record that a commit timestamp orders committed transactions across the parent and its components, with its evidence status as a target, so that every design resting on it is found by its `constrainedBy` edge (F18) ### Open questions -- [non-blocking] The internal runtime upper-bound method is absent from the public server declarations. A supported public upper bound, ordering under overlapping transactions and behavior on a hosted deployment are outside the native example; no platform consumer adopts the internal method by this fact. (F18) +- [non-blocking] The internal runtime upper-bound method is absent from the public server declarations. A supported public upper bound, ordering under overlapping transactions and behavior on a hosted deployment are outside the native example; no platform consumer adopts the internal method by this fact (F18, Probe 8) ## Constraints -- statement: On the pinned native backend, a transaction writes the same resolved bigint commit timestamp in the parent and two components, later sequential transactions have larger timestamps, and a top-level mutation may return its placeholder as the resolved bigint; within the mutation a read yields the unresolved placeholder and scheduling that placeholder is refused. (F18) +- statement: On the pinned native backend, a transaction writes the same resolved bigint commit timestamp in the parent and two components, later sequential transactions have larger timestamps, and a top-level mutation may return its placeholder as the resolved bigint; within the mutation a read yields the unresolved placeholder and scheduling that placeholder is refused (F18) - flavor: convex-fact - target: evidence.status:probed -- measurableBy: Compiled tier: `convex` 1.46.0, `node_modules/convex/dist/esm-types/server/database.d.ts:308-318` declares the placeholder and `:199` keeps the schema type on insert, and `node_modules/convex/dist/esm-types/server/meta.d.ts:125-136` declares no snapshot timestamp method; package source `node_modules/convex/src/server/meta.ts:137-160` describes the internal `getSnapshotTs()` upper bound, and `node_modules/convex/src/server/impl/meta_impl.ts:56-59,78,94` supplies it at runtime; `node_modules/convex/src/values/value.ts:98-127` defines unresolved numeric conversion; native backend tier: Probe 8 on the pinned release. (F18) +- measurableBy: Compiled tier: `convex` 1.46.0, `node_modules/convex/dist/esm-types/server/database.d.ts:308-318` declares the placeholder and `:199` keeps the schema type on insert, and `node_modules/convex/dist/esm-types/server/meta.d.ts:125-136` declares no snapshot timestamp method; package source `node_modules/convex/src/server/meta.ts:137-160` describes the internal `getSnapshotTs()` upper bound, and `node_modules/convex/src/server/impl/meta_impl.ts:56-59,78,94` supplies it at runtime; `node_modules/convex/src/values/value.ts:98-127` defines unresolved numeric conversion; native backend tier: Probe 8 on the pinned release (F18) ## Design -- nativeOrder: Probe 8 commits three sequential transactions, each writing the parent and two component tables indexed by `commitTs`; each transaction has one timestamp across all three tables and the indexed query returns increasing timestamps. Rows in one transaction tie, so a timestamp alone is not a unique event cursor. (F18, Probe 8) -- nativeUpperBound: A cast exposing the internal runtime `ctx.meta.getSnapshotTs()` returns the same upper bound in the parent and both component queries; every observed row is at or below it and a subsequent mutation's returned timestamp is above it. This is native backend evidence for the internal method, not a compiled public API promise. (F18, Probe 8) -- placeholderBehavior: The writing mutation reads back the same placeholder and numeric conversion throws that it is unresolved. Returning the placeholder from a mutation resolves it to the committed row's bigint. Scheduling it fails with `Field name $commitTs starts with '$', which is reserved.` and rolls back the accompanying write. (F18, Probe 8) -- firstObservation: The expectation that returning a mutation's placeholder would fail was refuted on the pinned native backend. The example binds the observed resolved bigint; refusal to schedule held. (F18, Probe 8) -- schemaBoundary: The fixture uses `v.any()` for its timestamp field and asserts the committed value is a bigint. The pinned writer's insert declaration takes the schema's field type and does not widen an int64 field to accept `CommitTsPlaceholder`. (F18, Probe 8) +- nativeOrder: Probe 8 commits three sequential transactions, each writing the parent and two component tables indexed by `commitTs`; each transaction has one timestamp across all three tables and the indexed query returns increasing timestamps. Rows in one transaction tie, so a timestamp alone is not a unique event cursor (F18, Probe 8) +- nativeUpperBound: A cast exposing the internal runtime `ctx.meta.getSnapshotTs()` returns the same upper bound in the parent and both component queries; every observed row is at or below it and a subsequent mutation's returned timestamp is above it. This is native backend evidence for the internal method, not a compiled public API promise (F18, Probe 8) +- placeholderBehavior: The writing mutation reads back the same placeholder and numeric conversion throws that it is unresolved. Returning the placeholder from a mutation resolves it to the committed row's bigint. Scheduling it fails with `Field name $commitTs starts with '$', which is reserved.` and rolls back the accompanying write (F18, Probe 8) +- firstObservation: The expectation that returning a mutation's placeholder would fail was refuted on the pinned native backend. The example binds the observed resolved bigint; refusal to schedule held (F18, Probe 8) +- schemaBoundary: The fixture uses `v.any()` for its timestamp field and asserts the committed value is a bigint. The pinned writer's insert declaration takes the schema's field type and does not widen an int64 field to accept `CommitTsPlaceholder` (F18, Probe 8) diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md index 74aa3c3..52ddb4e 100644 --- a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md @@ -7,24 +7,24 @@ relations: refines: spec:facts.f19-nested-calls-share-time-budgets verifies: spec:facts.f19-nested-calls-share-time-budgets --- -# Component calls per mutation +# Probe 11: component calls per mutation -Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. +Probe 11 · native tier · fixture composition. ## Intent -- outcome: Establish the observed behavior on the pinned local backend. +- outcome: A mutation that repeats empty component calls is refused with the system-operation error after a count the run records, and an empty call reads and writes no document. (Probe 11, F19) ```gwt Given a parent mutation that calls an empty component mutation repeatedly When the client increases the call count until the backend refuses the mutation -Then the error names {budget: "Your request timed out performing too many system operations."} time +Then the refused mutation's error is {budget: "Your request timed out performing too many system operations."} And each empty component call reads {reads: 0} and writes {writes: 0} documents ``` ## Verification — executable -- Every test owns a disposable native backend. Timings and call counts are recorded, not asserted, and say nothing about a hosted deployment. -- The first native run confirmed the expected distinction between system-operation time and execution time. The example binds the full observed error text. A run that never reaches a refusal fails. -- Successful call counts, the first refused requested count, wall time and top-level completion time are recorded under the stated load. The failed request cannot return its exact completed-call count. +- Runs in the native tier on the fixture composition; every test owns its disposable backend. Times and call counts are recorded, not asserted, and say nothing about a hosted deployment. +- The expectation written before the first run, a refusal distinct from the one-second execution limit, held; the example binds the full error text that run showed. A run that never reaches a refusal fails. +- Successful call counts, the first refused requested count, wall time and top-level completion time are recorded in the run's evidence, marked as taken under load unless `LIBAR_TIMINGS_UNDER_LOAD=0` says the machine was otherwise idle. The failed request cannot return its exact completed-call count. - A twenty-call control also inserts and reads one document per component call, asserts one read and one write per call, and records transaction metrics and completion usage. The example verifies F19 only; it does not compare against a plain helper for F4. diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md index f3737b6..361944f 100644 --- a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md @@ -7,23 +7,23 @@ relations: refines: spec:facts.f19-nested-calls-share-time-budgets verifies: spec:facts.f19-nested-calls-share-time-budgets --- -# Nested calls reach a time boundary +# Probe 9: nested calls reach a time boundary -Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. +Probe 9 · native tier · fixture composition. ## Intent -- outcome: Establish the observed behavior on the pinned local backend. +- outcome: A mutation that repeats empty nested calls is refused with the system-operation error, and a loop of computation is refused with the one-second execution error. (Probe 9, F19) ```gwt Given a parent mutation that calls an empty nested mutation repeatedly When the client increases the call count until the backend refuses the mutation -Then the error names {budget: "Your request timed out performing too many system operations."} time -And a direct computation fails with {cpu: "Function execution timed out (maximum duration: 1s)"} time +Then the refused mutation's error is {budget: "Your request timed out performing too many system operations."} +And a loop of computation in one mutation fails with {cpu: "Function execution timed out (maximum duration: 1s)"} ``` ## Verification — executable -- Every test owns a disposable native backend. Timings and call counts are recorded, not asserted, and say nothing about a hosted deployment. -- The first native run confirmed the expected distinction between system-operation time and execution time. The example binds the full observed error text. A run that never reaches a refusal fails. -- Successful call counts, the first refused requested count, wall time and top-level completion time are recorded under the stated load. The failed request cannot return its exact completed-call count. +- Runs in the native tier on the fixture composition; every test owns its disposable backend. Times and call counts are recorded, not asserted, and say nothing about a hosted deployment. +- The expectation written before the first run, a refusal distinct from the one-second execution limit, held; the example binds the full error text that run showed. A run that never reaches a refusal fails. +- Successful call counts, the first refused requested count, wall time and top-level completion time are recorded in the run's evidence, marked as taken under load unless `LIBAR_TIMINGS_UNDER_LOAD=0` says the machine was otherwise idle. The failed request cannot return its exact completed-call count. diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md index 356856a..15ce284 100644 --- a/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md @@ -6,29 +6,31 @@ readiness: defined relations: refines: spec:facts.fact-ledger --- -# Nested calls share time budgets +# Nested calls draw on a system-operation time budget -F19 · Native backend tier · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. +F19 · Status: probed · Doc status: not in the doc's ledger · Decisions: D2, D7, D10. + +A nested or component call runs through an asynchronous system call of the pinned package, and the pinned backend does not charge the time a mutation spends waiting on its calls to the one second of execution F13 lists. It charges it to a budget of system operations, whose refusal has its own error. Probe 9 and Probe 11 ran on the native backend, release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0: a mutation that made a thousand empty calls committed after several seconds, and one that made more was refused about fifteen seconds after it began, in every run; a loop of computation was refused at one second with the execution error. The pinned package declares no duration and no metric for the budget, so the count of calls one mutation may make is not a number, and it falls as each call does more. The runs were taken while other work shared the machine, and their times are recorded and not asserted. ## Intent -- outcome: Record the package declaration and native backend behavior with the limits of the evidence. (F19) +- outcome: Record that the calls one mutation makes draw on a system-operation time budget beside the one-second execution limit, with its evidence status as a target, so that every design resting on it is found by its `constrainedBy` edge (F19) ### Open questions -- [non-blocking] The public package declares no duration for the system-operation budget. The local timeout near fifteen seconds is an observation under load, not a hosted limit. The examples exercise mutations; query-only call trees and the allocation of JavaScript execution time among busy child functions remain unmeasured. (F19) +- [non-blocking] The public package declares no duration for the system-operation budget. The local refusal near fifteen seconds is a local backend's observation, not a hosted limit. The examples exercise mutations; query-only call trees and how execution time is counted across child functions that compute are not measured (F19, Probe 9, Probe 11) ## Constraints -- statement: On the pinned native backend, repeated empty nested mutation calls and component mutation calls in one parent mutation can exceed one second of top-level elapsed execution and end with a system-operation timeout, distinct from the one-second execution timeout of direct computation. (F19) +- statement: On the pinned native backend, the nested and component calls of one mutation can take it past one second of elapsed time without reaching the one-second execution limit, and end in a system-operation refusal with its own error (F19) - flavor: convex-fact - target: evidence.status:probed -- measurableBy: Package source: `convex` 1.46.0, `node_modules/convex/src/server/impl/registration_impl.ts:63-68,755-770` sends nested and component mutation calls through the asynchronous `1.0/runUdf` syscall; `node_modules/convex/dist/esm-types/server/meta.d.ts:19-27` exposes byte and document metrics but no time metric; native backend tier: Probe 9 and Probe 11 on the pinned release, whose errors name the system-operation budget and the one-second execution budget. (F19) +- measurableBy: Package source: `convex` 1.46.0, `node_modules/convex/src/server/impl/registration_impl.ts:63-68,755-770` sends nested and component mutation calls through the asynchronous `1.0/runUdf` syscall; `node_modules/convex/dist/esm-types/server/meta.d.ts:19-27` exposes byte and document metrics but no time metric; native backend tier: Probe 9 and Probe 11 on the pinned release, whose errors name the system-operation budget and the one-second execution limit (F19) ## Design -- systemBoundary: A mutation that repeatedly calls an empty nested or component mutation fails with `Your request timed out performing too many system operations.` while its completion record reports zero documents and bytes read and written. A direct computation fails with `Function execution timed out (maximum duration: 1s)`. Both errors are bound, while elapsed times and requested call counts are recorded. (F19, Probe 9, Probe 11) -- nativeMeasurements: Under load, an initial local run completed 1,000 nested calls in 3.751 seconds and 1,000 component calls in 7.083 seconds; requests for 10,000 calls failed at 15.703 and 15.169 seconds respectively. These are top-level completion times on one local backend, not limits asserted by the examples. (F19, Probe 9, Probe 11) -- componentCost: Probe 11 compares transaction metrics before and after twenty component calls and records the top-level completion usage and time. Empty calls read and write zero documents. Each control call inserts one document and reads it back, costing one document read and one written; the initial run recorded 13.133 ms per empty call and 15.521 ms per control call under load. (F19, Probe 11) -- countBoundary: The probe increases the requested count until the backend refuses it, recording every successful count and the first failed request. The timeout can end execution before JavaScript catches the error, so the failed request's exact completed-call count is not observable by the return value. No count is bound as a platform limit. (F19, Probe 11) -- factCoverage: Probe 11 verifies this time-budget fact. It records component-call costs that inform the concern of F4, but does not verify F4's comparison against a plain helper because it has no helper baseline. (F19, F4, Probe 11) +- systemBoundary: A mutation that repeatedly calls an empty nested or component mutation fails with `Your request timed out performing too many system operations.` while its completion record reports zero documents and bytes read and written. A loop of computation fails with `Function execution timed out (maximum duration: 1s)`. Both errors are bound, while elapsed times and requested call counts are recorded (F19, Probe 9, Probe 11) +- nativeMeasurements: With other work on the machine, a thousand empty nested calls committed in 3.8 to 9.4 seconds and a thousand empty component calls in 5.0 to 7.1 seconds of top-level completion time, and every refused request ended between 15.2 and 15.7 seconds; these are one local backend's times under load, recorded in each run's evidence and asserted by no example (F19, Probe 9, Probe 11) +- componentCost: Probe 11 compares transaction metrics before and after twenty component calls and records the top-level completion usage and time. Empty calls read and write zero documents. Each control call inserts one document and reads it back, costing one document read and one written; a call took several milliseconds under load, and the times are recorded and not asserted (F19, Probe 11) +- countBoundary: The probe increases the requested count until the backend refuses it, recording every successful count and the first failed request. The refusal can end execution before JavaScript catches the error, so the failed request's exact completed-call count is not observable by the return value. No count is bound as a platform limit (F19, Probe 11) +- factCoverage: Probe 11 verifies this time-budget fact. It records component-call costs that inform the concern of F4, but does not verify F4's comparison against a plain helper because it has no helper baseline (F19, F4, Probe 11) diff --git a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md index 014c195..5b095c1 100644 --- a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md +++ b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md @@ -7,23 +7,23 @@ relations: refines: spec:facts.f20-pagination-checks-bytes-after-reading verifies: spec:facts.f20-pagination-checks-bytes-after-reading --- -# A page keeps the row that crosses its byte bound +# Probe 10: a page keeps the row that crosses its byte bound -Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. +Probe 10 · native tier · fixture composition. ## Intent -- outcome: Establish the observed behavior on the pinned local backend. +- outcome: A page read under a byte bound below one row's size, or equal to it, still reads and keeps that row and comes back `SplitRequired`. (Probe 10, F20) ```gwt Given a component list with two rows larger than the requested byte bound When the parent reads the component paginator with byte bounds below and at one row Then the first page keeps {rows: 1} row and reports {status: "SplitRequired"} -And the recorded bytes exceed the smaller requested bound by {overshoot: true} +And the bytes read are above every bound below the row's size {overshoot: true} ``` ## Verification — executable -- Every test owns a disposable native backend. Timings and call counts are recorded, not asserted, and say nothing about a hosted deployment. -- The bound values are the expectation written before the first native run. A run that never reaches its boundary fails. +- Runs in the native tier on the fixture composition; every test owns its disposable backend. The stored size of the row and the bytes each page read, from `ctx.meta.getTransactionMetrics()`, are recorded in the run's evidence. +- The bound values are the expectation written before the first run, which held. The bounds are zero, one, one byte below the stored size and the stored size; a page that kept no row would not have reached the boundary, and the test fails it. - The test also supplies a negative byte bound. The helper still reads and keeps one row, so the overshoot guarantee is stated only for finite non-negative bounds. diff --git a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md index 446ccb3..ec0c34b 100644 --- a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md +++ b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md @@ -7,23 +7,24 @@ relations: refines: spec:facts.f20-pagination-checks-bytes-after-reading verifies: spec:facts.f20-pagination-checks-bytes-after-reading --- -# A page includes stored system fields in its byte check +# Probe 10: a page of full stream rows reads past the library's byte bound -Probe 10 · Native backend tier · Fixture composition · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. +Probe 10 · native tier · fixture composition. ## Intent -- outcome: Read rows at their declared application-field budget through the context library's page options and record the system-field bytes beyond the page bound. +- outcome: Rows that fill the default stream budget in application fields, read to an end cursor with the context library's page options, carry the page past the library's byte bound by their system fields. (Probe 10, F20) ```gwt Given {count: 32} component rows whose application fields each occupy {budgetBytes: 262144} bytes When the component paginator reads to the end cursor using the context library's page options Then the page keeps {kept: 32} rows and marks {status: "SplitRequired"} -And its measured bytes are above the library byte bound by {exceeds: true} +And the bytes the page read are above the library's byte bound {exceeds: true} ``` ## Verification — executable +- Runs in the native tier on the fixture composition; every test owns its disposable backend. - The test uses `boundedPage`, `limitListPage` and `limitListBytes` from `src/context/queries.ts`, with an end cursor that removes the item count as a stopping condition. -- The bound values are the expectation written before the first native run. Stored sizes and measured bytes are recorded, not bound as constants. -- The table has the same application-field budget as a default single-mapping stream. Its stored rows also carry `_id` and `_creationTime`, which the helper includes in the byte count. +- The bound values are the expectation written before the first run. Stored sizes and measured bytes are recorded, not bound as constants. +- The table has the same application-field budget as a default single-mapping stream. Its stored rows also carry `_id` and `_creationTime`, which the helper includes in the byte count. The row cap of twice the item cap is reached on the same row as the byte bound, so the example shows what the page read and not which cap stopped it. diff --git a/design/specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md b/design/specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md index 7102247..eadc3d5 100644 --- a/design/specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md +++ b/design/specs/facts/f20-pagination-checks-bytes-after-reading.sdp.md @@ -8,25 +8,27 @@ relations: --- # Pagination checks bytes after reading -F20 · Native backend tier · Backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124. +F20 · Status: probed · Doc status: not in the doc's ledger · Decisions: D8, D10. + +`paginator` of `convex-helpers` 0.1.124 adds each row's size to the bytes it has read after it has read and kept the row, and only then compares the sum with `maximumBytesRead`. So the option is a threshold at which a page stops, not a ceiling on what the page reads: a page reads the row that reaches the threshold, whatever its size. Probe 10 ran it on the native backend, release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, through a component's `paginator` as a context's list reads it. The size the helper adds is the stored document's, `_id` and `_creationTime` included, so rows that fill a budget counted in application fields carry a page past a byte bound computed from those budgets. ## Intent -- outcome: Record the package declaration and native backend behavior with the limits of the evidence. (F20) +- outcome: Record that `paginator` checks `maximumBytesRead` after it has read the row that reaches it, with its evidence status as a target, so that every design resting on it is found by its `constrainedBy` edge (F20) ### Open questions -- [non-blocking] The bound applies to a plain index stream. Filtered, joined or mapped streams may charge reads whose rows are not retained, and the examples do not establish an overshoot bound for those compositions. A hosted deployment is not measured. (F20) +- [non-blocking] The bound applies to a plain index stream. Filtered, joined or mapped streams may charge reads whose rows are not retained, and the examples do not establish an overshoot bound for those compositions. A hosted deployment is not measured (F20, Probe 10) ## Constraints -- statement: For a plain component index stream with a finite non-negative byte bound, convex-helpers keeps the next row and adds its stored document size before comparing the total with maximumBytesRead using greater-than-or-equal; a page can therefore keep and read beyond that bound by up to the last row's size and then report SplitRequired. (F20) +- statement: For a plain component index stream with a finite non-negative byte bound, convex-helpers keeps the next row and adds its stored document size before comparing the total with maximumBytesRead using greater-than-or-equal; a page can therefore keep and read beyond that bound by up to the last row's size and then report SplitRequired (F20) - flavor: convex-fact - target: evidence.status:probed -- measurableBy: Package source: `convex-helpers` 0.1.124, `node_modules/convex-helpers/server/stream.js:292-315` for the late check and `:513-525` for `getDocumentSize`; native backend tier: Probe 10 on the pinned release; `src/context/queries.ts:49,100` passes `maximumBytesRead` through `boundedPage` and `spec:context.queries` is the Spec that rests on this fact. (F20) +- measurableBy: Package source: `convex-helpers` 0.1.124, `node_modules/convex-helpers/server/stream.js:292-315` for the late check and `:513-525` for `getDocumentSize`; native backend tier: Probe 10 on the pinned release; `src/context/queries.ts:49,100` passes `maximumBytesRead` through `boundedPage` and `spec:context.queries` is the Spec that rests on this fact (F20) ## Design -- nativeOvershoot: With two stored rows of 8,281 bytes each, native queries with byte bounds of zero, one, 8,280 and 8,281 each read and keep the first row and report `SplitRequired`; the bound of zero does not mean read nothing, and equality also requests a split. The size is recorded, not asserted as a release limit. (F20, Probe 10) -- storedFields: `getDocumentSize` includes `_id` and `_creationTime`. A page computed from application-field budgets can exceed the claimed whole-page byte size even if each row's application fields fits its budget; the stream-budget example uses the actual `boundedPage`, `limitListPage` and `limitListBytes` helpers to measure that case. (F20, Probe 10) -- scopeOfBound: The last row is retained, so this option is a stopping threshold rather than a hard maximum of bytes already read or returned. A transaction that begins close to its backend read ceiling can fail while fetching that row before the helper gets to check the threshold. (F20, Probe 10) +- nativeOvershoot: With two stored rows of 8,281 bytes each, native queries with byte bounds of zero, one, 8,280 and 8,281 each read and keep the first row and report `SplitRequired`; the bound of zero does not mean read nothing, and equality also requests a split. The size is recorded, not asserted as a release limit (F20, Probe 10) +- storedFields: `getDocumentSize` includes `_id` and `_creationTime`. A page computed from application-field budgets can exceed the claimed whole-page byte size even if each row's application fields fits its budget; the stream-budget example uses the actual `boundedPage`, `limitListPage` and `limitListBytes` helpers to measure that case (F20, Probe 10) +- scopeOfBound: The last row is retained, so this option is a stopping threshold rather than a hard maximum of bytes already read or returned. A transaction that begins close to its backend read ceiling can fail while fetching that row before the helper gets to check the threshold (F20, Probe 10) diff --git a/design/specs/facts/fact-ledger.sdp.md b/design/specs/facts/fact-ledger.sdp.md index a09dfb2..f0492e9 100644 --- a/design/specs/facts/fact-ledger.sdp.md +++ b/design/specs/facts/fact-ledger.sdp.md @@ -43,7 +43,7 @@ The four statuses map to the constraint target `evidence.status:documented`, `ev - factsRechecked: F3, F4, F5, F6 on 2026-09-29; F16 on 2026-09-30 against S6, changed from assumed (Fact ledger, F16) - factsAssumed: F17, with Probe 6; the doc also lists F14 and F15, which the corpus records as probed (Fact ledger, Probes) - factsProbed: F14 by Probe 2 and F15 by Probe 5, on 2026-10-01 on backend release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0; Probe 1 on F5, Probe 3 on F4 and Probe 4 on F13 ran the same day, and their results are recorded on those facts, whose status stays documented or rechecked (Fact ledger, Probes) -- additionalFactsProbed: F18 by Probe 8, F19 by Probe 9 and Probe 11, and F20 by Probe 10 on native backend `precompiled-2026-09-28-5c7cb5b`, with `convex` 1.46.0 and `convex-helpers` 0.1.124; each states its package source and the limits of the local evidence. (Fact ledger, F18, F19, F20) +- factsBeyondTheDoc: F18 by Probe 8, F19 by Probe 9 and Probe 11, and F20 by Probe 10 on native backend `precompiled-2026-09-28-5c7cb5b`, with `convex` 1.46.0 and `convex-helpers` 0.1.124; each states its package source and the limits of the local evidence (Fact ledger, F18, F19, F20) - sourcesRead: S1 to S13 by v0.1's author on 2026-09-29, S10, S14 and S15 again during the review the same day, and S2, S6, S8, S9, S10, S11 and the migrations and application-errors pages by the lead on 2026-09-30 (Sources) ## Verification — reviewed diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index c7ef8ba..1aa4f79 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -25,6 +25,8 @@ Feature · Detail: full transcription · Traces: Probe 1 to Probe 7, F4, F5, F12 Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`, so Probe 7 need only confirm them and answer what a restore leaves; the backup page says nothing about component data or the scheduler, so that half of Probe 7 stands. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, and each result is recorded on its fact; probes 6 and 7 have not run. +Probes 8 to 11 settle three facts the doc's ledger does not list, F18 to F20, each written from the pinned package source: the commit timestamp, the time budget nested calls draw on with the count of component calls one mutation may make, and the byte check of `convex-helpers`. + The probes are not the first experiment. The experiment builds Layers 0 to 2 and measures cost; the probes settle single facts and can run before it, in a fixture app that is separate from the example app. ## Intent @@ -65,11 +67,10 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - Probe 5 subscribes to a parent query that calls a component query, changes the component's data, and checks that the subscription updates; it then pages a component list built with `paginator` through the parent while live writes land inside the paged range, with the client passing the end cursor as `convex-helpers` describes, records whether the pages stay contiguous, and fills one page's range past the component's `maximumRowsRead` so that a re-run comes back `SplitRequired`, recording whether the `convex-helpers/react` hook splits it across the boundary without a gap; it serves D8 (Probe 5, F15, S4) - Probe 6 runs a backfill batch through the migrations component while a live command writes the same read-model row, and checks that neither overwrites newer data and that the batch resumes after interruption; it serves D9 (Probe 6, F1, F17) - Probe 7, before Layer 3, takes a backup with pending scheduled functions, Workpool jobs and a Workflow run in flight, restores it, and records what survives; it reads `_scheduled_functions` for a failed run and confirms the states and the retention window; it serves D13 and D19 (Probe 7, F12, F16) - -- Probe 8 writes commit timestamp placeholders in the parent and two components, compares committed rows and the runtime upper bound, and checks readback, return and scheduling behavior. (Probe 8, F18) -- Probe 9 repeats nested calls until a time boundary fails and compares its error with direct computation. (Probe 9, F19) -- Probe 10 reads component pages below and at a row's byte size and records the bytes kept beyond the requested bound. (Probe 10, F20) -- Probe 11 increases component calls per mutation to the backend's refusal, records counts and time, and measures documents read and written. (Probe 11, F19) +- Probe 8 writes commit timestamp placeholders in the parent and two components, compares committed rows and the runtime upper bound, and checks readback, return and scheduling behavior (Probe 8, F18) +- Probe 9 repeats empty nested calls in one mutation until the backend refuses it, and compares that error with the one a loop of computation gets (Probe 9, F19) +- Probe 10 reads component pages below and at a row's byte size and records the bytes kept beyond the requested bound (Probe 10, F20) +- Probe 11 increases component calls per mutation to the backend's refusal, records counts and time, and measures documents read and written (Probe 11, F19) ## Design diff --git a/tests/native/f19-nested-calls-share-time-budgets.probe-11-component-calls.test.ts b/tests/native/f19-nested-calls-share-time-budgets.probe-11-component-calls.test.ts index 09e7ce6..1d915bf 100644 --- a/tests/native/f19-nested-calls-share-time-budgets.probe-11-component-calls.test.ts +++ b/tests/native/f19-nested-calls-share-time-budgets.probe-11-component-calls.test.ts @@ -27,7 +27,7 @@ bindExample(contract, (): BudgetWorld => ({}), { ) => prepareBudget(world, "component"), "the client increases the call count until the backend refuses the mutation": reachBudget, - "the error names {budget} time": (world, { budget }) => { + "the refused mutation's error is {budget}": (world, { budget }) => { expect(world.error).toContain(budget); }, "each empty component call reads {reads} and writes {writes} documents": diff --git a/tests/native/f19-nested-calls-share-time-budgets.probe-9-nested-time.test.ts b/tests/native/f19-nested-calls-share-time-budgets.probe-9-nested-time.test.ts index d5d03f8..2e41172 100644 --- a/tests/native/f19-nested-calls-share-time-budgets.probe-9-nested-time.test.ts +++ b/tests/native/f19-nested-calls-share-time-budgets.probe-9-nested-time.test.ts @@ -27,9 +27,11 @@ bindExample(contract, (): BudgetWorld => ({}), { ) => prepareBudget(world, "nested"), "the client increases the call count until the backend refuses the mutation": reachBudget, - "the error names {budget} time": (world, { budget }) => { + "the refused mutation's error is {budget}": (world, { budget }) => { expect(world.error).toContain(budget); }, - "a direct computation fails with {cpu} time": async (world, { cpu }) => - computeBudget(world, cpu), + "a loop of computation in one mutation fails with {cpu}": async ( + world, + { cpu }, + ) => computeBudget(world, cpu), }); diff --git a/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.test.ts b/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.test.ts index 2cb596d..e555b28 100644 --- a/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.test.ts +++ b/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.test.ts @@ -80,7 +80,7 @@ bindExample(contract, (): World => ({}), { expect(read.result.pageStatus).toBe(status); } }, - "the recorded bytes exceed the smaller requested bound by {overshoot}": ( + "the bytes read are above every bound below the row's size {overshoot}": ( world, { overshoot }, ) => { diff --git a/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.test.ts b/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.test.ts index 8ff04d4..4dd3b0c 100644 --- a/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.test.ts +++ b/tests/native/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.test.ts @@ -61,7 +61,7 @@ bindExample(contract, (): World => ({}), { ) => { expect(world.result).toMatchObject({ count: kept, status }); }, - "its measured bytes are above the library byte bound by {exceeds}": ( + "the bytes the page read are above the library's byte bound {exceeds}": ( world, { exceeds }, ) => { diff --git a/tests/native/first-experiment-steps.ts b/tests/native/first-experiment-steps.ts index b43420a..9b78c31 100644 --- a/tests/native/first-experiment-steps.ts +++ b/tests/native/first-experiment-steps.ts @@ -77,10 +77,6 @@ export interface Placed { window: CompletionRecord[]; summaryKeys: string[]; } -const scheduledByOrder = new WeakMap< - Placed, - Awaited> ->(); export interface ExperimentWorld { backend?: Backend; client?: ConvexHttpClient; @@ -204,7 +200,7 @@ export async function place( ), "the command's completion record", ); - const placed: Placed = { + return { orderId, lines, response, @@ -213,9 +209,13 @@ export async function place( window, summaryKeys: summaries.page.map((row) => row.key), }; - scheduledByOrder.set(placed, await scheduledRows(backend)); - return placed; } +// The scheduled-function tables of the parent and both contexts, read after the commands: a job +// scheduled with any delay is a row there. +const scheduledAfter = new WeakMap< + ExperimentWorld, + Awaited> +>(); export async function placeOrderRuns( world: ExperimentWorld, run: "the PlaceOrder use case" | "the end-to-end path", @@ -230,6 +230,10 @@ export async function placeOrderRuns( "order-1", orderLines(linesOf[size], "sku"), ); + scheduledAfter.set( + world, + await scheduledRows(required(world.backend, "the backend")), + ); } const sent = (world: ExperimentWorld): Placed[] => [world.reference, required(world.placed, "the order")].filter( @@ -251,12 +255,17 @@ export function commitsAre(world: ExperimentWorld, commits: number) { expect(placed.request).toEqual(committed); } } -// Admin reads of all scheduled-function tables include delayed jobs. The ordinary query already -// shows the command's summary row. +// Nothing ran between the command and the read sent after it returned, nothing is scheduled to run +// later, and that read already shows the command's summary row: the command wrote it, and no job did. export function projectionJobsAre(world: ExperimentWorld, jobs: number) { for (const placed of sent(world)) { + const others = placed.window.filter( + (record) => + record !== placed.own && record.identifier !== summariesIdentifier, + ); + expect(others).toHaveLength(jobs); expect( - required(scheduledByOrder.get(placed), "the scheduled-function rows"), + required(scheduledAfter.get(world), "the scheduled-function rows"), ).toHaveLength(jobs); expect(placed.summaryKeys).toContain(placed.orderId); } diff --git a/tests/native/read-models.committed-state-read-through-query.test.ts b/tests/native/read-models.committed-state-read-through-query.test.ts index a1f3961..18b29bd 100644 --- a/tests/native/read-models.committed-state-read-through-query.test.ts +++ b/tests/native/read-models.committed-state-read-through-query.test.ts @@ -29,7 +29,7 @@ const anchor = specTest({ void anchor; // PlaceOrder commits in its own mutation, and the parent query getOrder, which relays the Orders // context's get, answers the committed order as soon as the command has returned. The function log -// identifies both client calls. Admin reads of the scheduled-function tables check that no job exists. +// from the command to that read shows that nothing but the two calls the client made ran. type Response = FunctionReturnType; interface World { order?: OrderWorld; @@ -38,6 +38,9 @@ interface World { } const orderId = "order-1"; const lines = [line("sku-1", 2, 250), line("sku-2", 1, 40)]; +// A record a client caused: a call over HTTP, or a query or mutation over the socket. A scheduled +// function, a cron or an action is a worker. +const clientCallers = new Set(["HttpApi", "SyncWorker"]); const placeOrderIdentifier = "ordering:placeOrder"; const getOrderIdentifier = "orderQueries:getOrder"; bindExample(contract, (): World => ({}), { @@ -95,7 +98,8 @@ bindExample(contract, (): World => ({}), { version: orderVersions[0], }); }, - // The log identifies the two client calls; the tables include delayed scheduled functions. + // Every record in the function log from the command to the read: the command's own mutation and + // the read's query, both caused by the client. "the number of workers, jobs or queues that ran is {workers}": async ( world, { workers }, @@ -107,6 +111,16 @@ bindExample(contract, (): World => ({}), { entries.some((entry) => entry.identifier === placeOrderIdentifier) && entries.some((entry) => entry.identifier === getOrderIdentifier), ); + expect( + records.filter( + (entry) => + entry.udfType === "Action" || + entry.udfType === "HttpAction" || + !clientCallers.has(entry.caller), + ), + ).toHaveLength(workers); + // The scheduled-function tables of the parent and both contexts also hold a job scheduled with + // a delay past the window. expect(await scheduledRows(backend)).toHaveLength(workers); expect( records.find((entry) => entry.identifier === placeOrderIdentifier), diff --git a/tests/native/read-models.committed-state-visible.test.ts b/tests/native/read-models.committed-state-visible.test.ts index 55ba1d8..f78189a 100644 --- a/tests/native/read-models.committed-state-visible.test.ts +++ b/tests/native/read-models.committed-state-visible.test.ts @@ -30,7 +30,7 @@ const anchor = specTest({ void anchor; // The order summary is written by PlaceOrder's own mutation, in step 9 of the pipeline. A client // subscribed to the summary list sees the row the moment the command commits, and the function log -// identifies those calls. Admin reads of the scheduled-function tables check that no job exists. +// shows that nothing but the command and the subscription's query ran in between. type Summaries = FunctionReturnType; type Response = FunctionReturnType; interface World { @@ -41,6 +41,9 @@ interface World { } const orderId = "order-1"; const lines = [line("sku-1", 2, 250), line("sku-2", 1, 40)]; +// A record a client caused: a call over HTTP, or a query or mutation over the socket. A scheduled +// function, a cron or an action is a worker. +const clientCallers = new Set(["HttpApi", "SyncWorker"]); bindExample(contract, (): World => ({}), { "a read model maintained by the {useCase} use case": async ( world, @@ -119,7 +122,8 @@ bindExample(contract, (): World => ({}), { }, ]); }, - // The log identifies the two client calls; the tables include delayed scheduled functions. + // Every record in the function log from the command to the subscription's update: the command's + // own mutation and the subscription's query, both caused by the client. "the number of workers, jobs or queues that ran is {workers}": async ( world, { workers }, @@ -133,6 +137,16 @@ bindExample(contract, (): World => ({}), { (entry) => entry.identifier === "readModels:listOrderSummaries", ), ); + expect( + records.filter( + (entry) => + entry.udfType === "Action" || + entry.udfType === "HttpAction" || + !clientCallers.has(entry.caller), + ), + ).toHaveLength(workers); + // The scheduled-function tables of the parent and both contexts also hold a job scheduled with + // a delay past the window. expect(await scheduledRows(backend)).toHaveLength(workers); expect( records.find((entry) => entry.identifier === "ordering:placeOrder"), From c385a37777c4e5deda8d4d1b5e53bbe145951cb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:23:47 +0200 Subject: [PATCH 017/155] design: the mechanical read of the generations and gate text, and its ledger A read of the Spec text for what would make a test and a build disagree. Each fix follows the text the designer decided: the progress row gets its update time at insertion, the batch that ends a pass records its counts, an empty tenant list still reads the gate, a purged generation drops its retirement fields, the history batch replaces steps 4 and 5, the restore's exemption names the progress rows `resumeChain` writes, a closed entry reopens only at its generation's switch or abort, and the stale sentences on the first activation, the tenant iteration and the audit tenant rule follow the text. The ledger records what the text closes and what the build must still show. Co-Authored-By: Claude Opus 5.5 --- design/reviews/consensus-ledger.json | 82 ++++++++++--------- design/specs/application.pack.sdp.md | 2 +- ...ctivation-makes-read-model-writable.sdp.md | 2 +- .../application/generation-registry.sdp.md | 8 +- design/specs/application/rebuild.sdp.md | 18 ++-- ...build.write-pause-rebuild-interrupt.sdp.md | 2 +- design/specs/application/restore.sdp.md | 4 +- design/specs/application/write-pause.sdp.md | 8 +- design/specs/command/command-pipeline.sdp.md | 2 +- .../operations/baseline-operations.sdp.md | 8 +- design/specs/platform/native-harness.sdp.md | 2 +- 11 files changed, 70 insertions(+), 68 deletions(-) diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index 7b5abc2..badccf3 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -1917,7 +1917,7 @@ "issue": "A read model's binding carries one projection at one version: `ReadModelBinding = { name; version; source; projection }` (command-declaration typeReadModelBinding). The pipeline's step 9 reads `generationsToWrite`, which returns generation numbers only, and calls `applyProjection(ctx, projection, { ..., generations })` once with every generation to write (projection-contract fnApplyProjection, callers). So the active generation, built under the previous projection version, and the building one both receive the output of the one projection the deployed code holds. The contract says 'a new version is a new generation, never an in-place rewrite' and rebuild repeats it as a rule, yet from the deploy that changes a projection, before `startGeneration` is even called, every live command rewrites the active generation's rows in place with the new version's output and stamps the new `projectionVersion` on them. Queries then read a generation that mixes two row shapes until the switch. Rollback has the same hole from the other side: `rollbackGeneration` reopens the retired generation, 'live commands write it again', and they write it with the new projection, so the generation the operator rolls back to is no longer the old build. Sc L2-5's 'cutover and rollback work' holds only when the projection did not change, which is the one case the generation mechanism is not for. No Spec says which projection code writes which generation. A second gap sits beside it: `startGeneration`, the batches, the restore's inventory and its read-model check take a read model by name, but its projection and sources live only on command declarations, repeated on every command that maintains it, and no Spec says how a name resolves to them.", "fix": "Pin one read-model registry in the parent, one constant per read model: its name, its sources and its projections keyed by version. Command declarations list read-model names or reference the constant, so the version and the projection are stated once (D12). `generationsToWrite` returns `{ generation, projectionVersion }` pairs and step 9 writes each generation through the projection whose version the generation row records; the registration check fails when a generation in `active`, `building`, `verifying`, `verified` or `retired` inside its rollback period names a version the registry no longer holds. State the standing cost under E-8 or E-9: the previous projection stays in the code until its generation is purged. Make the L2-5 example rebuild with a changed projection version and assert that the active generation's rows keep the old version until the switch and that the rolled-back generation is written by the old projection.", "status": "open", - "note": "the platform review of 2026-10-02, APPL-01: At HEAD, generation-registry.sdp.md:72 already returns each generation with \"projectionVersion\" and command-declaration.sdp.md:66 references one ReadModel, so the issue's old numbers-only and repeated-binding clauses are stale, but projection-contract.sdp.md:92 still stamps \"the declared projection's version\" on every row and can mix v1 and v2. Keep the version-selected projection work and old-version retention, adapting the fix to the existing ReadModel constant rather than repeating wiring that is already shared. The build must show: convex-test must keep active v1 and building v2 rows in their own shapes under live creates and updates, and a native backend rollout must keep reads valid and return to v1 after intervening writes. A guard-only cut must refuse mismatch before any write and state the supported deployment procedure.", + "note": "the platform review of 2026-10-02, APPL-01: At HEAD, generation-registry.sdp.md:72 already returns each generation with \"projectionVersion\" and command-declaration.sdp.md:66 references one ReadModel, so the issue's old numbers-only and repeated-binding clauses are stale, but projection-contract.sdp.md:92 still stamps \"the declared projection's version\" on every row and can mix v1 and v2. Keep the version-selected projection work and old-version retention, adapting the fix to the existing ReadModel constant rather than repeating wiring that is already shared. The build must show: convex-test must keep active v1 and building v2 rows in their own shapes under live creates and updates, and a native backend rollout must keep reads valid and return to v1 after intervening writes. A guard-only cut must refuse mismatch before any write and state the supported deployment procedure. S4 on paper, 2026-10-02. `spec:application.projection-contract` pins `projections`, the latest first, and `projectionOf`, and its `step1` and `step4` write each generation with the projection whose version its row records; `spec:application.rebuild` refuses an undeclared version at `startGeneration` and `rollbackGeneration`, and the E-9 question states the standing cost. No check at registration is stated: a generation whose version the read model no longer declares fails at the write, through `projectionOf`. The build must show: at convex-test, an active generation of version 1 and a building one of version 2 each keep their own rows under live creates and updates; on the native backend, a rollout to version 2 and a return to version 1 after intervening writes.", "slice": "S4", "detector": "convex-test" }, @@ -1968,8 +1968,8 @@ ], "issue": "`rollbackGeneration` takes any read model: it sets the retired generation to `verifying`, 'live commands write it again from that commit', and a verify pass repairs it. For a history view nothing closes the sources: write-pause says no function other than `startGeneration` closes a source scope for a rebuild. So after a rollback the next live command folds its new events onto a retired row that never saw the events between the switch and the rollback, and `applyHistoryProjection` raises that row's `sourceVersions` to the new event's version. The missed events are now 'not above the recorded version' and are dropped by every later fold, and `historyVerify`, which only compares the last event's version, passes. The rolled-back generation becomes active with holes in it and no check says so. projection-contract's claim that 'the live path never meets a non-active generation of a history view because its building generation exists only while its source scopes are closed' is false on this path, and rebuild's own `rebuildClass` bullet says why it matters: 'a live fold onto a row backfill has not reached cannot be told apart from a complete one'.", "fix": "Rule one way under E-8. Either `rollbackGeneration` refuses a `pauseRequired` generation with a plain error, and rolling a history view back means a new paused generation at the old projection version; or it closes every source scope of the view in the same mutation, naming the reopened generation, runs its catch-up pass under the pause, and the scopes reopen only after the switch, exactly as a first build does. Correct the projection-contract sentence to match, and add the rollback of the history view, or its refusal, to the L2-6 examples.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-05: At HEAD, rebuild.sdp.md:108 sets a retired generation to \"verifying\" without closing its sources, while projection-contract.sdp.md:72 drops events below the row watermark, so v10 can hide missed v7-v9. The issue and refusing rollback or reacquiring the write pause remain right. The build must show: convex-test must retire at v6, commit v7-v9 and race v10 with rollback, then repeat on the native backend; all relevant effects must be present or v10 refused while the required pause remains held.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-05: At HEAD, rebuild.sdp.md:108 sets a retired generation to \"verifying\" without closing its sources, while projection-contract.sdp.md:72 drops events below the row watermark, so v10 can hide missed v7-v9. The issue and refusing rollback or reacquiring the write pause remain right. The build must show: convex-test must retire at v6, commit v7-v9 and race v10 with rollback, then repeat on the native backend; all relevant effects must be present or v10 refused while the required pause remains held. S4 on paper, 2026-10-02. `spec:application.rebuild` `fnRollbackGeneration` refuses a generation with `pauseRequired`, and the E-8 question on what the workflow does not offer says why. `spec:application.rebuild.write-pause-rebuild-interrupt` still binds 'can roll back', for roadmap 4c, which writes the paused path; the build must show the refusal at convex-test.", "slice": "S4", "detector": "convex-test" }, @@ -1986,7 +1986,7 @@ "issue": "The unfixed half of r2-architecture-history-and-cross-context-projection-has-no-input. The contract still says a cross-context row 'holds one `StreamVersion` per context' and that such a row spreads `rowConventions`, read-models offers 'one cross-context read-model row' as a read, and the declaration says 'a read model fed by several stream types is bound once per source with the same projection'. For a `Projection⟨D, Row⟩` that cannot work: `project(tenantId, dto, versions)` builds a whole row from one DTO, and `applyProjection` writes it with `ctx.db.insert` or `ctx.db.replace`. Bound to two sources, the second entry's row replaces the first, so the row holds one context's fields and one context's version. No form takes two DTOs or merges into a prior row, and the backfill enumerates one source. The only form that can hold two contexts is `HistoryProjection`, which rebuilds only under the write pause, although D9 says a current-state read model rebuilds online. An implementer who wants D8's 'small cross-context view' as a read model has to guess between a history view with a pause it does not need and a form the corpus does not define.", "fix": "Take one of the two options the round-2 finding gave. Narrow: state in projection-contract and read-models that Layer 2 pins per-entity, aggregate and history forms, that a small cross-context current-state view is D8's parent query over bounded component reads, and remove 'cross-context row' from the `Projection` bullets, the row conventions and `fnCrossContextView`; record the missing composite form as a [non-blocking] E-9 question. Or add `CompositeProjection`: `project` takes the DTOs of every matching `streams` entry of one command plus the prior row, the write compares `sourceVersions` per stream, and the backfill composes one `list` DTO with bounded `get` reads of the other context in the same mutation, online.", "status": "open", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: A read-model row needs current DTOs from two contexts, but the declaration and live writer only pass one stream entry at a time; the unsupported generic promise has no complete write form. The build must show: Compiled contracts and convex-test must either provide the complete source-read/write form and compare live output with rebuild after either source changes, or explicitly narrow the supported promise to the per-subject path until an actual consumer requires it.", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: A read-model row needs current DTOs from two contexts, but the declaration and live writer only pass one stream entry at a time; the unsupported generic promise has no complete write form. The build must show: Compiled contracts and convex-test must either provide the complete source-read/write form and compare live output with rebuild after either source changes, or explicitly narrow the supported promise to the per-subject path until an actual consumer requires it. S4 on paper, 2026-10-02. Narrowed in `spec:application.rebuild`, the E-8 question on what the workflow does not offer: a row built from more than one source stream type has no rebuild there. Its write form on the live path, in `spec:application.projection-contract` and `spec:application.read-models`, is not designed.", "slice": "S4", "detector": "convex-test" }, @@ -2040,7 +2040,7 @@ "issue": "The round-2 ruling that a generation spans every tenant made the rebuild depend on 'the application's tenant list, read in tenant-ID order': `startGeneration` puts the cursor 'at the first tenant' of it, `tenantIteration` moves to 'the next tenant' of it, the restore's checks under `all` iterate it 'the way the rebuild's `tenantIteration` does', the baseline driver runs 'one chain per tenant' because 'the parent, not the component, knows the tenants', and the receipt sweep runs per tenant. No Spec defines that list. The parent's pinned tables are `grants`, `receipts`, `generations`, `projectionMarkers`, `maintenanceGates`, `imports`, `restoreRuns`, `auditRecords` and `personalData`; none enumerates tenants, and `grants` has no index that yields distinct tenant IDs. The L2-5 and L2-8 examples seed two tenants and assert the cursor crosses them, so the first experiment cannot build its rebuild without inventing the table, its index, and what happens to a tenant created while a pass is under way.", "fix": "Pin a `tenants` table in the parent on `spec:command.actor-and-scope` or `spec:command.tenancy-and-authority`: `tenantId`, `createdAt`, an index on `tenantId`, written by the command or operator function that creates a tenant, as an [extension] with its register row. State in `tenantIteration` that a pass reads it by index after the cursor's tenant ID, one tenant per step, and rule on a tenant created behind the cursor: on the online path its streams are all created after `startGeneration` and so written live, and the verify pass enumerates again; under a write pause no stream can be created in a closed source. Point the restore's scope, the baseline driver's `driverTenancy` and the receipt sweep at the same table.", "status": "open", - "note": "the platform review of 2026-10-02, APPL-10: At HEAD, rebuild.sdp.md:103 requires \"the application's tenant list\" without declaring its authority or enumeration, so an all-tenant pass cannot locate its next tenant. The issue and structured enumeration or shared compare-helper fix remain right; completion must require every required check passed. The build must show: convex-test and native backend enumeration must visit every required tenant and exercise a tenant created behind the cursor under the ruled online or paused semantics, without treating absent tenant scope as a wildcard.", + "note": "the platform review of 2026-10-02, APPL-10: At HEAD, rebuild.sdp.md:103 requires \"the application's tenant list\" without declaring its authority or enumeration, so an all-tenant pass cannot locate its next tenant. The issue and structured enumeration or shared compare-helper fix remain right; completion must require every required check passed. The build must show: convex-test and native backend enumeration must visit every required tenant and exercise a tenant created behind the cursor under the ruled online or paused semantics, without treating absent tenant scope as a wildcard. S4 on paper, 2026-10-02. `spec:command.actor-and-scope` pins the `tenants` table, `insertGrant` writing a tenant's row and `nextTenant` (E-49); `spec:application.rebuild` `step2`, `step6` and the rule on a tenant behind the cursor iterate it, and `spec:application.restore` `validatorCheckScope` points at `nextTenant`. The receipt sweep of `spec:command.receipt-table` does not name it yet, for the unit that builds the sweep. The build must show at convex-test and on the native backend: two tenants and the cursor crossing, an empty tenant, and a tenant inserted behind the cursor.", "slice": "S4", "detector": "judgement" }, @@ -2078,7 +2078,7 @@ "issue": "The native tier is 'a disposable local backend per test' and the contract says nothing about how a test controls time or interrupts work, yet the examples depend on both. Interruption: L2-5 'kills the backend's scheduler after the third batch', L2-8 'stops the scheduler after its first batch', L2-6 'interrupts after the second history batch', L1-3 has 'the harness drops the response'. A local backend offers no scheduler switch, and a restart leaves the pending batch to run by itself, which exercises none of `resumeGeneration`. Time: L3-2 'advances past the grace' of 120 seconds on a 60 second cron, L3-5 'advances past the lease' of 35 minutes, L3-8 'advances 31 days', L3-4 needs eight attempts under a backoff that sums to minutes. No Spec makes any of these constants configurable and every one is read from `Date.now()` inside the function. The first experiment needs the interruptions to pass Layer 2, so it cannot be run from the Specs without inventing a harness, and the Layer 3 examples cannot run in bounded time at all.", "fix": "Add a harness contract to `spec:platform.acceptance-contract` as an [extension]. Interruption is a fault flag in the fixture app, a row a named batch or wrapper reads and throws on, never a scheduler kill, so the chain ends the way a developer error ends it and the operator path is what resumes it; a lost response is a caller that discards the result. Time: every duration the design pins, grace, cron interval, lease, backoff, deferral, retention horizons, rollback period, receipt window, is read from one module configuration with the pinned defaults, and a native run that shortens them records 'adjusted configuration' with the values, which the evidence rule already requires. Say which examples run with shortened values and that Sc L2-9's production-configuration run excludes them.", "status": "partially-fixed", - "note": "S0, 2026-10-01. `spec:platform.acceptance-contract` now names the native tier's controls under E-15: the backend process (start on fresh storage, stop, restart on the same storage), the clients (close, discard a response), the environment variables and, on the fixture composition, fixture functions. It has no clock control and no scheduler switch. Two claims in the issue are too broad: three durations are already configuration or arguments (receipt retention per command, backoff per handler, the rollback period), and the sweeper measures elapsed time with performance.now(). The fix's 'one module configuration' is not taken: whether the durations Layer 3 pins become configuration is left to Layer 3 and sits in the owner queue, and a lease cannot be shortened apart from the action timeout it must exceed. Replacing every interruption with an injected throw is not taken either: L3-2 separates a cancelled dispatch from a failed wrapper. No Layer 0 to 2 example needs time to pass. Still open: the Layer 2 examples that say 'kills the backend's scheduler' or 'stops the scheduler' (L2-5, both L2-6 siblings, L2-8) name a control that does not exist. S4 and S5 reword each to a listed control when they build it, and the build must show `resumeGeneration` run after an interruption that leaves no pending batch. The Layer 3 to 6 examples that advance time wait for Layer 3.", + "note": "S0, 2026-10-01. `spec:platform.acceptance-contract` now names the native tier's controls under E-15: the backend process (start on fresh storage, stop, restart on the same storage), the clients (close, discard a response), the environment variables and, on the fixture composition, fixture functions. It has no clock control and no scheduler switch. Two claims in the issue are too broad: three durations are already configuration or arguments (receipt retention per command, backoff per handler, the rollback period), and the sweeper measures elapsed time with performance.now(). The fix's 'one module configuration' is not taken: whether the durations Layer 3 pins become configuration is left to Layer 3 and sits in the owner queue, and a lease cannot be shortened apart from the action timeout it must exceed. Replacing every interruption with an injected throw is not taken either: L3-2 separates a cancelled dispatch from a failed wrapper. No Layer 0 to 2 example needs time to pass. Still open: the Layer 2 examples that say 'kills the backend's scheduler' or 'stops the scheduler' (L2-5, both L2-6 siblings, L2-8) name a control that does not exist. S4 and S5 reword each to a listed control when they build it, and the build must show `resumeGeneration` run after an interruption that leaves no pending batch. The Layer 3 to 6 examples that advance time wait for Layer 3. S4 on paper, 2026-10-02. The rebuild's share: `spec:application.rebuild` `fnInterruptGeneration` ends a chain on purpose by raising the fence, and `spec:application.rebuild.online-rebuild-interrupt-resume` interrupts through it once at least 3 batches are done on the native backend and after exactly 3 at convex-test, with no scheduler kill; `spec:platform.native-harness` lists a composition's operator entries among what admin access calls. The restore example of L2-8, the obligation and effect examples and the Layer 3 durations stay.", "slice": "S4", "detector": "judgement", "unreviewed": true @@ -2225,10 +2225,11 @@ ], "issue": "E-43 pins an `auditRecords` table that fails closed and does not weigh what Convex already gives, which decision method rule 2 asks of every mechanism. Convex has audit logging: 'Use `log.audit(params)` inside your Convex function', 'Function commit will block on log persistence', delivered 'to a S3 bucket in your own AWS account' (docs.convex.dev/production/integrations/audit-logging, read 2026-10-01). The same page gives the reasons it does not replace the table: it 'requires a Convex Enterprise plan and is only available for dedicated (D1024 and D2048) deployments', and it emits 'on mutations, both on success and failure, and possibly on OCC retries', so an entry does not prove a commit and the operator's queries by operation, subject and request key have nothing to read. The option and those reasons are not on the Spec.", "fix": "Add the native option to the E-43 open question as the do-nothing option with the two reasons it is not taken, and note that a deployment on a dedicated Enterprise plan may add `log.audit` beside the table as an export, never instead of it.", - "status": "open", - "note": "", + "status": "fixed", + "note": "S4 on paper, 2026-10-02. `spec:operations.baseline-operations` open question E-43 records Convex's own audit logging, `audit(body)` of the pinned `convex` package, as the do-nothing option, not taken because the operator's queries by operation, subject and request key cannot read a log outside the database and the pinned package does not show that an entry proves a commit, and lets a deployment add it beside the tables. The plan condition and the emission on failure and retry that the finding quotes from the docs page are not stated, because the Spec takes this fact from the pinned package; a reviewer confirms that this is enough.", "slice": "S4", - "detector": "judgement" + "detector": "judgement", + "unreviewed": true }, { "round": 3, @@ -2257,7 +2258,7 @@ "issue": "A verify batch 'checks that every stream whose DTO projects to a row has one', 'while a stream whose DTO projects to `null` needs no row'. On a first build that is enough, because a live delete removes a row that exists. After `rollbackGeneration` the retired generation still holds the row of a subject deleted between the switch and the rollback, the verify pass does not look at it, and the reopened generation becomes active showing a subject that no longer projects to a row.", "fix": "Have verify call `applyProjection` in backfill mode for a DTO that projects to `null` as well, which deletes the row if present and counts it as a repaired miss, and say so in the verify flow and in the L2-5 example's rollback step.", "status": "open", - "note": "the platform review of 2026-10-02, APPL-06: At HEAD, rebuild.sdp.md:81 says a DTO projecting to null \"needs no row\", and :108 reopens the retired generation for verification, so rollback can retain and disclose the retired row of a deleted subject. Keep the absence-repair fix and raise minor to major for returned-data correctness. The build must show: convex-test and native backend rollback must leave a subject deleted during retirement absent from the restored generation and its list subscription after switch, with source enumeration including deleted subjects.", + "note": "the platform review of 2026-10-02, APPL-06: At HEAD, rebuild.sdp.md:81 says a DTO projecting to null \"needs no row\", and :108 reopens the retired generation for verification, so rollback can retain and disclose the retired row of a deleted subject. Keep the absence-repair fix and raise minor to major for returned-data correctness. The build must show: convex-test and native backend rollback must leave a subject deleted during retirement absent from the restored generation and its list subscription after switch, with source enumeration including deleted subjects. S4 on paper, 2026-10-02. `spec:application.rebuild` `verifySteps` and `spec:application.projection-contract` `step1` and the backfill-mode rule delete the row of a DTO that projects to `null`, so one verify pass repairs absence as well as presence. The build must show at convex-test a subject deleted between the switch and the rollback absent from the reopened generation after its verify pass; the native step of `spec:application.rebuild.online-rebuild-interrupt-resume` allows a cancellation in place of a deletion, so the native share of absence is still to be shown.", "slice": "S4", "detector": "convex-test" }, @@ -2483,8 +2484,8 @@ ], "issue": "The doc's assumed fact F17 is '`@convex-dev/migrations` fits a generation backfill' and D9 says the component 'provides resumable batching'. The corpus states the component's reach two ways. journal, as a fact: '`@convex-dev/migrations` iterates a table of the deployment that defines the migration and cannot page a component's `streams` table from the parent (S4, E-25)'. rebuild, as an open question: 'whether a migration can iterate a table inside a component is what Probe 6 shows', with the self-scheduled batch used 'until Probe 6 shows the component can'. Every read model's source is a context component, so if the journal's reading of S4 is right the component drives no backfill in this design, only the purge and re-projection of parent rows, and F17 is answered in the negative for the case D9 means. None of that reaches the foundation Specs: f17's narrative says only that the race and the batch size are 'not stated', d09 lists 'the migrations component' as standing cost and repeats '`@convex-dev/migrations` provides resumable batching (D9, F17)', and probe-plan's Probe 6 flow 'runs a backfill batch through the migrations component while a live command writes the same read-model row', which is not the batch the design runs. The probe as transcribed would pass or fail without testing the self-scheduled batch that Sc L2-5 depends on.", "fix": "Settle the sentence once on `spec:facts.f17-migrations-fits-generation-backfill`: record in its narrative what S4 says about a component's tables, that the context-sourced backfill therefore runs as a self-scheduled mutation under E-40, and what is left for Probe 6. Make journal's `driverNotMigrationsComponent` and rebuild's assumption and Probe 6 question say the same thing. On `spec:facts.probe-plan`, have Probe 6 race the self-scheduled batch as well as a migrations-driven parent-table batch, marked as the corpus's widening of the doc's probe. Add to d09's cost consequence that the component is installed for parent-table batches only, so the owner can weigh dropping it.", - "status": "open", - "note": "", + "status": "partially-fixed", + "note": "S4 on paper, 2026-10-02. spec:application.rebuild E-40 and purgeDriver distinguish self-scheduled context enumeration from the future parent-table driver. spec:facts.f17-migrations-fits-generation-backfill, spec:facts.probe-plan, spec:context.journal and spec:decisions.d09-rebuild-online-by-default still need the common statement and probe alignment from their lanes.", "slice": "S4", "detector": "judgement" }, @@ -2604,8 +2605,8 @@ ], "issue": "The `grants` table is read by every command at step 4 and has no writer anywhere in the corpus. `tableGrants` carries `grantedBy: v.string(), grantedAt: v.number()`, tenancy-and-authority says grants 'are revoked by deleting the row', and the L1-8 example says 'The caller's grant is then deleted', but no Spec names the function that inserts or deletes a grant, who may call it, which permission it needs, or how the first grant of a tenant comes to exist when every command needs one. The first experiment cannot run a single `PlaceOrder` without a grant for its test principal, and Sc L2-9's run registers no test-only function, so the write has to be a production function the Specs do not give. Two inputs of the same contract are also unnamed: `establishActor(ctx, serviceIssuers)` takes a set described only as 'a configured service issuer', with no statement of where the configuration lives, and it returns only `human` or `service` actors, so nothing in Layers 1 and 2 ever produces the `operator` actor that every rebuild, gate and restore function takes as an argument. This is the same kind of gap as r3-architecture-tenant-list-undefined and r3-architecture-example-domain-cannot-create-stock, at a different table.", "fix": "Pin in `spec:command.actor-and-scope`, as an extension under E-6 or E-37 with its register row: `grantPermission` and `revokePermission` as two commands through the pipeline with a declaration each, a permission such as `grants.manage`, `audit: { kind: \"security\" }`, and the bootstrap rule for a tenant's first grant, for example an internal mutation run at tenant creation by the function that writes the tenant row r3-architecture-tenant-list-undefined asks for. State where `serviceIssuers` is declared, a constant in the parent's configuration module, and how an operator actor is established. Say in `spec:application.first-experiment` that the experiment seeds its grants through that path, and count the commands in the example model's Verification bullet.", - "status": "partially-fixed", - "note": "S1, 2026-10-01, ruling R5. `spec:command.actor-and-scope` pins `insertGrant` and `revokeGrant` as plain helpers that register no function, and states that the application registers its own internal mutations over them and that a tenant's first grant is created by an operator who runs one with admin access; `spec:command.tenancy-and-authority` carries this under E-37 as a provisional reading for the owner. `publicCommand` takes `serviceIssuers` as an optional second parameter. The fixture registers `grants:grant` and `grants:revoke`; the native test `tests/native/tenancy-and-authority.revoked-then-retried.test.ts` revokes through `grants:revoke`. Still open: S2 says in `spec:application.first-experiment` how the experiment seeds its grants and where its `serviceIssuers` constant lives; S4 says how an operator actor is established, with the operator entry points. S2, 2026-10-02, fold-in after the build. The S2 part is built: `spec:application.first-experiment` says a run seeds its grants through the production composition's internal grant mutation with admin access and that the production composition names no service issuer; built as `example/convex/grants.ts`, and every native test on the production composition seeds its grants that way, `tests/native/production-composition-deploys.test.ts` among them. Left for S4, with the operator entry points: how an operator actor is established.", + "status": "fixed", + "note": "S1, 2026-10-01, ruling R5. `spec:command.actor-and-scope` pins `insertGrant` and `revokeGrant` as plain helpers that register no function, and states that the application registers its own internal mutations over them and that a tenant's first grant is created by an operator who runs one with admin access; `spec:command.tenancy-and-authority` carries this under E-37 as a provisional reading for the owner. `publicCommand` takes `serviceIssuers` as an optional second parameter. The fixture registers `grants:grant` and `grants:revoke`; the native test `tests/native/tenancy-and-authority.revoked-then-retried.test.ts` revokes through `grants:revoke`. Still open: S2 says in `spec:application.first-experiment` how the experiment seeds its grants and where its `serviceIssuers` constant lives; S4 says how an operator actor is established, with the operator entry points. S2, 2026-10-02, fold-in after the build. The S2 part is built: `spec:application.first-experiment` says a run seeds its grants through the production composition's internal grant mutation with admin access and that the production composition names no service issuer; built as `example/convex/grants.ts`, and every native test on the production composition seeds its grants that way, `tests/native/production-composition-deploys.test.ts` among them. Left for S4, with the operator entry points: how an operator actor is established. S4 on paper, 2026-10-02. As the main thread's ruling on the first grant has it, `spec:command.actor-and-scope` `grantWriters` and `spec:command.tenancy-and-authority` E-37 say a tenant's first grant is created with admin access through the composition's own internal grant mutation, later grants the same way or by a command the composition registers, and that the library registers no grant function and promises no recorded bootstrap. The operator actor the S2 note left for S4 is answered by E-48, which establishes no actor of kind `operator` and records a stated operator. `insertGrant` also writes the tenant list (E-49).", "slice": "S4", "detector": "judgement", "unreviewed": true @@ -2660,8 +2661,8 @@ ], "issue": "Every Layer 2 maintenance operation is an internal function that takes the acting operator as a plain argument, and the Specs lean on a wrapper that no Spec defines. rebuild's Design says 'All rebuild functions are `internalMutation` or `internalQuery` and reach the dashboard through operator functions that authorize an operator actor; no public function starts, switches or aborts a generation'. baseline-operations says 'Operator functions that read audit or diagnostics are internal functions reached through an operator actor'. write-pause says a resume during a build 'is refused by the operator function' and that scope keys are 'built only by `scopesOfUseCase`, `startGeneration`, the journal's baseline driver and the operator functions'. `startGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration`, `closeGate`, `resumeGate`, `startRestoreRun`, `finishRestoreRun` and `writeBaseline` all carry `actor: actorValidator` or `operator: actorValidator` in `args`. Nothing says what an operator function is: its registration kind, how the caller's identity becomes that argument, which permission is checked, or how the L2-5, L2-6 and L2-8 tests call it. Three pinned shapes make it impossible to fill in by analogy. `establishActor` 'returns ... a `service` actor when the identity's issuer is in `serviceIssuers`, and a `human` actor otherwise', so nothing produces an actor of kind `operator`. `AuthorizeInput = { tenantId: string; actor: Actor; permission: string; subject?: SubjectRef }` and every `grants` row carry a tenant, while a generation, the gate's `all` scope and a restore run are deployment-wide ('no registry function, batch or query takes a tenant scope for a generation'), so the authority to start a rebuild cannot be expressed as a grant. And a caller with the deployment's admin key can run any internal function with any `actor` value, so as pinned the argument is a label, not an authorization. Layer 3 at least pins a pattern, public `mutation` exports whose step1 is 'establish the actor from `ctx.auth` in the parent and authorize `obligations:repair`', but it has the same missing producer for the `operator` kind.", "fix": "Add one operator-entry contract, as an extension with its register row, to `spec:operations.baseline-operations` (operations from Layer 1) or `spec:command.actor-and-scope`. It rules three things. How an operator actor is established: for example a principal that holds an operator grant is mapped to kind `operator` by `establishActor`. How deployment-wide authority is stored, since an absent tenant is never a wildcard: a second small table of deployment grants, or grants under one reserved platform tenant, with the permission names listed (`rebuild.manage`, `gate.manage`, `restore.run`, `baseline.migrate`, `audit.read`). And what the entry is: either one public mutation or query per operation that authenticates, authorizes the named permission and then calls the internal function's helper in the same transaction with the actor it established, or a plain statement that Layer 2 operations are run from the dashboard or CLI under deployment admin rights and that `actor` is then an unverified label kept for the audit record, with the words 'authorize an operator actor' removed. Point rebuild, write-pause, restore, journal and `spec:obligations.operator-operations` at that contract, and say in the L2-5, L2-6 and L2-8 examples which entry the test calls.", - "status": "open", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: Starting a deployment-wide rebuild or restore takes an actor label at an internal function but names no authenticated operator entry or deployment-wide permission; an admin caller can supply any label. The build must show: Compiled entries and convex-test must establish the ruled operator identity and authority before maintenance, refuse missing rights without changes and distinguish an admin label from authenticated product authority; native backend ordinary-client tests must use the actual declared entry.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: Starting a deployment-wide rebuild or restore takes an actor label at an internal function but names no authenticated operator entry or deployment-wide permission; an admin caller can supply any label. The build must show: Compiled entries and convex-test must establish the ruled operator identity and authority before maintenance, refuse missing rights without changes and distinguish an admin label from authenticated product authority; native backend ordinary-client tests must use the actual declared entry. S4 on paper, 2026-10-02. `spec:command.actor-and-scope` E-48 pins the operator entry: an internal function only admin access reaches, a required `operator` string checked by `assertOperator` before any read, no grant read, and the operator recorded on what it changes (`operatorRecordedWhere`); `spec:application.rebuild`, `spec:application.write-pause` and `spec:application.restore` take it on every entry, and 'authorize an operator actor' is gone. The main thread ruled that the five repair operations of `spec:obligations.operator-operations` keep their tenant grant and left the baseline driver of `spec:context.journal` to the unit that builds baselines; the repair operations' sentence on an actor of kind `operator` established by the parent still disagrees with E-48. The build must show at the pure test tier `assertOperator` on an empty, a whitespace-only and an overlong operator, and on the native backend the examples calling the entries with admin access.", "slice": "S4", "detector": "judgement" }, @@ -2726,7 +2727,7 @@ "issue": "The audit path fails closed, and neither of its two writers can build the row the table requires. `fnWriteAudit` is `writeAudit(ctx: MutationCtx, record: AuditRecordInput)` and `AuditRecordInput` is defined nowhere. `tableAuditRecords` requires `operationId`, `commandType`, `subject: v.object({ contextId, streamType, streamId })`, `kind` and `decision: v.string()`. The pipeline's step 10 calls it 'for a declaration that sets `audit`', and the declaration gives only `audit?: { kind: \"security\" | \"business\" }`: `subject` has no source when the optional `permission.subjectFrom` is absent or when a use case touches many streams, as `PlaceOrder` with 101 does, `decision` has no stated values, and `causedBy` is `v.optional(v.string())` while the operation's cause is the `CausedBy` union. The second writer is the gate: write-pause says 'every close, resume and abort writes an audit record inside the same mutation and fails closed', and `closeScope` and `resumeScope` have a scope key, a reason and an actor, with no operation ID, no command type and no stream subject. baseline-operations says of the write that 'any failure, including a validator refusal or a document too large, throws and rolls the whole command back', so a wrong guess here refuses every audited command, and a gate close that cannot write its record refuses every paused rebuild.", "fix": "In `spec:operations.baseline-operations` define `AuditRecordInput` and give each field its source under E-43: `subject` is `decl.permission.subjectFrom(input)` when declared and otherwise the first affected ref, `decision` is the outcome kind, `causedBy` is the operation's cause in a stated string form, `requestKey` and `actor` come from the `PipelineCall`. Make the table fit an operator action: either `operationId`, `commandType` and `subject` become optional with a third `kind` of `operator` that carries `action` and `scopeKey`, or gate changes go to their own small table; say which in write-pause `fnCloseScope` and `fnResumeScope`. Repeat the field sources in the pipeline's `auditWrite` bullet so step 10 can be written from it.", "status": "open", - "note": "", + "note": "S4 on paper, 2026-10-02. `spec:operations.baseline-operations` `typeAuditRecordInput`, `auditRecordSources`, `tableAuditRecords`, `tableOperatorAudit` and `fnWriteOperatorAudit` pin both records and the source of every field; `spec:application.write-pause` `fnCloseScope` and `fnResumeScope` write the gate's. The build must show at the compiled tier that `AuditRecordInput` is the table's document without `recordedAt`, and at convex-test step 10's record with each field from its source, both subject sources, and the fail-closed case with neither.", "slice": "S4", "detector": "TypeScript compiler" }, @@ -2746,7 +2747,7 @@ "issue": "No Spec says how a read model gets its first active generation, and without one the command path writes nothing. The pipeline's step 9 writes 'the generations `generationsToWrite` returns', which is `{ active?: number; building?: number }` read from the `generations` table; on a fresh deployment that table is empty, so the answer is empty and no row is written, and the registry adds 'a read model with no active generation reads as empty'. The only way into `active` is the listed chain 'building to verifying, verifying to verified, verified to active', that is `startGeneration`, the backfill and verify batches and `switchGeneration`, each an operator call that also needs the tenant list of r3-architecture-tenant-list-undefined. Every rebuild example starts from 'a read model with an active generation', the L2-2 example runs `PlaceOrder` on its own disposable backend and asserts 'the next subscription value's page carries the order's summary row', and `limitExpectedDocuments` counts 'the summary row' in every measured `PlaceOrder`. None of them says who created generation 1. An implementer has to choose between running a full rebuild of an empty read model as a deploy step, which no release rule mentions, and inventing a bootstrap write that the transition list forbids.", "fix": "Rule it in `spec:application.generation-registry` under E-8: either the deploy that first declares a read model inserts generation 1 directly as `active` with the declared projection version, through a named internal mutation that refuses when any generation row of that read model exists, and the transition list gains 'none to active for the first generation'; or `generationsToWrite` treats a read model with no row as generation 1 active and writes the row on first use, with the cost of that write stated. Add the step to `releaseRule` in `spec:operations.baseline-operations` and to the build flows of `spec:application.first-experiment`, and have the L2-2 example say how its backend got the active generation.", "status": "partially-fixed", - "note": "S2, 2026-10-02, fold-in after the build. The first generation is one explicit transition, the first activation, run with admin access and legal only when the read model has no generation row (`spec:application.generation-registry`); built as `activateFirstGeneration` and each composition's `readModels:activate`. Shown at the native tier on the production composition by `tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts` and `tests/native/generation-registry.command-without-generation-fails.test.ts`, and at the convex-test tier by `tests/simulator/read-model.test.ts`; every production test runs the activation in its setup. A binding admits only the per-entity read model (ruling C4), so the first activation is defined for that form alone: the build that takes Sc L2-6 must say how a history and an aggregate read model are bound and first activated. Left: that, for S4, and `releaseRule` in `spec:operations.baseline-operations`, which does not name the first activation as a deploy step, for S5.", + "note": "S2, 2026-10-02, fold-in after the build. The first generation is one explicit transition, the first activation, run with admin access and legal only when the read model has no generation row (`spec:application.generation-registry`); built as `activateFirstGeneration` and each composition's `readModels:activate`. Shown at the native tier on the production composition by `tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts` and `tests/native/generation-registry.command-without-generation-fails.test.ts`, and at the convex-test tier by `tests/simulator/read-model.test.ts`; every production test runs the activation in its setup. A binding admits only the per-entity read model (ruling C4), so the first activation is defined for that form alone: the build that takes Sc L2-6 must say how a history and an aggregate read model are bound and first activated. Left: that, for S4, and `releaseRule` in `spec:operations.baseline-operations`, which does not name the first activation as a deploy step, for S5. S4 on paper, 2026-10-02. `spec:application.generation-registry` installs a read model through its first rebuild: generation 1 starts as `building` and no transition leads from nothing to active; its example `first-activation-makes-read-model-writable` and the three `spec:application.read-models` examples set up that way, and the example keeps its ID until the build renames it with its test. Still a first activation in `spec:application.first-experiment` and its three order-size examples, another lane's files, with the lines given to the main thread; `spec:operations.baseline-operations` `releaseRule` does not say that a release installs a new read model through its first rebuild.", "slice": "S4", "detector": "convex-test" }, @@ -2813,7 +2814,7 @@ "issue": "Small things in Layers 0 to 2 that are named and not pinned, each a short guess for the implementer. (1) `Bounds = { maxItems?: number; maxBytes?: number }` is checked 'when the input's items exceed `bounds.maxItems`', and nothing says which part of an input is its items; the L2-3 maximum example needs it to mean `lines`. (2) The pipeline lists parse as step 1 and authenticate as step 2, and parent-use-cases says the bound is 'checked by the pipeline's step 1 before authentication', but `runPipeline` takes a `PipelineCall` whose `actor` is required and the public helper 'establishes the actor ... and calls `runPipeline`', so for the public entry authentication runs first. (3) outcome-boundary says 'a code outside both is a technical failure by definition', while `normalizeThrown(error, commandType)` rethrows any bare kernel `Rejection` as a rejection and never sees the declaration's `rejections`; `irreversible` likewise 'requires' a tombstone with no check named. (4) receipt-table says 'a declaration may set a finite horizon' for `tombstoneHorizon`, and `Retention = { window; afterExpiry }` has no field for it. (5) `limitRequestKey` says a longer key 'is rejection `invalidInput` at step 1'; Step 1 and `errorCodeInvalidInput` do not mention it. (6) `canonicalJson` is defined for JSON values only; a Convex input may hold `v.int64()` or `v.bytes()`. (7) `get` returns `null` for a deleted subject 'unless the DTO policy shows deleted subjects', and no field declares that policy; `list` takes `includeDeleted` and its handler text never uses it. (8) `Journal` is the return type of `createJournal` with no members; `streamParts.streamVersion` has no rule for what is written to it or who reads it. (9) `carryOrReject(outcome, policy)` throws 'through the outcome boundary's `reject`', which requires `commandType`, and takes its `message` from nowhere. (10) `applyHistoryProjection` takes `mode: ApplyMode` and no step reads it; the `events` it is fed include `baseline` events, and the contract does not say `keyOf` must return `null` for them. (11) Validators and types used once and never given: `generationDoc`, `gateDoc`, `AuditRecordInput`, `DiagnosticSink`, `consoleSink`, and in Layer 3 `callInputValidator`, `obligationViewValidator`, `dispatchResultValidator`. (12) One bound has two names, `limitGrantsRead` in actor-and-scope and `limitGrantsPerPrincipal` in tenancy-and-authority, and the handler registry has two shapes, `HandlerRegistry` mapping to a `FunctionReference` in obligation-module while `createStep3` reads `entry.maxAttempts` from the wrapper's `HandlerEntry`. (13) `convexTypeImport` says a value import of `convex/values` 'is a type error'; `verbatimModuleSyntax` rejects only a type imported without `import type`, so `import { v }` compiles and only the dependency check catches it. (14) `limitExpectedDocuments` lists the reads of one `PlaceOrder` and leaves out the grants read of step 4. (15) The experiment's examples rely on 'the production `auth.config.ts`' and no Spec names its provider or issuer.", "fix": "One line each in the owning Spec: add `itemsOf?: (input: I) => number` to `Bounds` in `spec:command.command-declaration`, or say `maxItems` counts the elements of the input's one top-level array; state in the pipeline that the public entry authenticates before step 1's bound and refinement checks, and correct the parent-use-cases sentence; pass `decl.rejections` to `normalizeThrown` and rethrow an unlisted code as a plain error, or drop the 'technical failure by definition' sentence, and name the registration check as the place `irreversible` is checked; add `tombstoneHorizon?: number` to `Retention`; add the key-length check to Step 1; define `canonicalJson` over `convexToJson(value)`; add `showDeleted?: boolean` to `StreamRegistration` or drop the clause, and say `list` filters deleted rows after the page is read unless `includeDeleted`; give `Journal` its two fields and `streamParts.streamVersion` its rule or remove the field; let `carryOrReject` throw the bare kernel `Rejection` and let `normalizeThrown` add the command type; say `mode` is ignored by `applyHistoryProjection` and that `keyOf` returns `null` for `baseline`; define the listed validators beside their one use; keep one name for the grants bound and one registry type; reword `convexTypeImport`; add the grants read to `limitExpectedDocuments`; name the experiment's auth provider in `spec:application.first-experiment`.", "status": "partially-fixed", - "note": "S1, 2026-10-01. Fixed in the Specs as built: (1) `maxItems` counts the elements of the input's top-level arrays; (2) the public entry authenticates before step 1's bound and refinement checks (ruling S3), and the parent-use-cases sentence follows; (3) `normalizeThrown(error, commandType, rejections)` rethrows an unlisted code as a plain error (ruling F2); (5) the key-length check is in step 1; (6) `canonicalJson` is defined over `convexToJson`; (8) `Journal` has its two fields. Shown by the convex-test test 'step 1 refuses a long request key and a refined title as invalidInput, and an input over its bounds as operationTooLarge' in `tests/simulator/command.test.ts`, the pure tests of `canonicalJson` and `normalizeThrown` in `tests/pure/command.test.ts` and 'a journal is the frozen configuration it was created with' in `tests/pure/context.test.ts`. Still open, by slice: S2 items (7), (9), (14) and (15); S4 item (10) and the operator validators of (11); S5 the `irreversible` check of item (3) and item (4) with tombstones; the slice that builds the derived mapping, `streamParts.streamVersion` of item (8); items (12) and (13) are polish. S2, 2026-10-02, fold-in after the build. S2 items: (7) fixed, `get` answers `null` for a deleted subject and `list` takes `includeDeleted` for a trusted caller (`spec:context.queries`), shown by `tests/simulator/context-queries.test.ts` (convex-test) and `tests/native/context-queries.test.ts` (native); (14) fixed, `limitExpectedDocuments` names the grants read, and the documents `PlaceOrder` reads on the pinned backend, 3N + 5 against the N + 6 expected, are an `observed` entry with an open question in `spec:application.first-experiment`, shown by the three Sc L2-3 native tests; (15) fixed, `authProvider` in `spec:application.first-experiment`, shown by `tests/native/first-experiment.native-acceptance-production-configuration.test.ts` (native, production composition); (9) fixed in the Spec only, `carryOrReject` throws the bare `Rejection`, and it waits for the build of `carryOrReject`, which no slice names yet. Left: S4 item (10) and the operator validators of (11); S5 the `irreversible` check of (3) and item (4); the derived mapping's `streamParts.streamVersion` of (8); (12) and (13) are polish.", + "note": "S1, 2026-10-01. Fixed in the Specs as built: (1) `maxItems` counts the elements of the input's top-level arrays; (2) the public entry authenticates before step 1's bound and refinement checks (ruling S3), and the parent-use-cases sentence follows; (3) `normalizeThrown(error, commandType, rejections)` rethrows an unlisted code as a plain error (ruling F2); (5) the key-length check is in step 1; (6) `canonicalJson` is defined over `convexToJson`; (8) `Journal` has its two fields. Shown by the convex-test test 'step 1 refuses a long request key and a refined title as invalidInput, and an input over its bounds as operationTooLarge' in `tests/simulator/command.test.ts`, the pure tests of `canonicalJson` and `normalizeThrown` in `tests/pure/command.test.ts` and 'a journal is the frozen configuration it was created with' in `tests/pure/context.test.ts`. Still open, by slice: S2 items (7), (9), (14) and (15); S4 item (10) and the operator validators of (11); S5 the `irreversible` check of item (3) and item (4) with tombstones; the slice that builds the derived mapping, `streamParts.streamVersion` of item (8); items (12) and (13) are polish. S2, 2026-10-02, fold-in after the build. S2 items: (7) fixed, `get` answers `null` for a deleted subject and `list` takes `includeDeleted` for a trusted caller (`spec:context.queries`), shown by `tests/simulator/context-queries.test.ts` (convex-test) and `tests/native/context-queries.test.ts` (native); (14) fixed, `limitExpectedDocuments` names the grants read, and the documents `PlaceOrder` reads on the pinned backend, 3N + 5 against the N + 6 expected, are an `observed` entry with an open question in `spec:application.first-experiment`, shown by the three Sc L2-3 native tests; (15) fixed, `authProvider` in `spec:application.first-experiment`, shown by `tests/native/first-experiment.native-acceptance-production-configuration.test.ts` (native, production composition); (9) fixed in the Spec only, `carryOrReject` throws the bare `Rejection`, and it waits for the build of `carryOrReject`, which no slice names yet. Left: S4 item (10) and the operator validators of (11); S5 the `irreversible` check of (3) and item (4); the derived mapping's `streamParts.streamVersion` of (8); (12) and (13) are polish. S4 on paper, 2026-10-02. The registry's, the gate's and audit's names are pinned: `generationDoc` and `progressDoc` of `fnGetGenerations`, `operatorAuditDoc` of `fnGetGateAudit`, the return of `fnGetGate`, `AuditRecordInput` and `OperatorAuditInput`, and the message of every rebuild and gate refusal. The other Specs' items stay.", "slice": "S4", "detector": "judgement", "unreviewed": true @@ -2838,8 +2839,8 @@ ], "issue": "Things the Specs define that no pinned function uses. (1) The `read-model:` scope: `scopesOfUseCase` emits one per maintained read model and `backfillBatch` passes 'the read-model scope' to `gateAllows`, but `closeGate` closes 'a tenant, a source stream type or the `all` scope' and `startGeneration` closes source scopes only, so nothing can ever close it. STATE lead 6 names this for the sdp lens. (2) `@convex-dev/migrations`: rebuild's rule makes it 'the batch runner ... where the batch's source is a parent table' and `migrationsBinding` gives 'purging a generation or re-projecting an existing generation's rows' as its uses, but the purge is `purgeGenerationBatch`, an internal mutation of its own with `limitPurgeBatch`, no function re-projects a generation, and every backfill enumerates a context through the self-scheduled chain. As pinned the component is mounted and runs nothing, yet D9's cost line, F17 and Probe 6 are carried for it. (3) The `by_event_id` index: tables gives its use as 'resolving a cross-context reference `(tenantId, contextId, eventId)`', and the query surface is `get`, `list`, `history`, `rebuild` and `byOperation`, with no query by event ID; the obligation payload `{ kind: \"eventRef\"; contextId; eventId; eventSchemaVersion }` therefore has no function that can load its event. (4) Indexes whose only stated use is 'operator inspection' or 'diagnosis' with no function behind them: `grants.by_permission`, `receipts.by_operation`, and the three audit queries `getAuditByOperation`, `getAuditBySubject`, `getAuditByRequestKey`, which are listed in `convexSurface` without signatures. (5) Fields with a writer and no reader: `StreamDto.appended` in the parent, `streams.updatedAt`, `streamParts.streamVersion`. (6) `applyAggregate`'s `mode`, which the Spec itself says is 'read by nothing'. (7) `typeCausation` in fan-out-and-chains is 'what a reaction body holds while it runs, from its `HandlerArgs`', and `HandlerArgs` has no such member.", "fix": "Remove the `read-model:` scope from `scopesOfUseCase`, `typeScopeKey` and the batches' gate read, or let `closeGate` close it and say what that is for. Decide whether the migrations component is mounted in Layers 0 to 2: either run the purge through it and delete `fnPurgeGenerationBatch`, or drop `migrationsBinding` and `migrationsState` and record under E-40 that the component is unused until Probe 6, which keeps F17 as a fact about a future option. Add `queries.events.byEventId` to `spec:context.queries`, or strike the `eventRef` payload and say the index waits for its first query, as D2's 'add others when a real query needs them' allows. Pin the operator queries with their signatures or drop the indexes that exist only for them. Drop the fields and the parameter nothing reads, or name the reader.", - "status": "open", - "note": "", + "status": "partially-fixed", + "note": "S4 on paper, 2026-10-02. spec:application.write-pause typeScopeKey removes read-model scopes; spec:application.rebuild purgeDriver removes the unused migrations binding and state; spec:operations.baseline-operations tableAuditRecords removes details. The source-generation exemption is still unmatched by rebuild batch scope reads. Event-reference reads, inspection signatures, unused fields and causation remain with their owning lanes.", "slice": "S4", "detector": "judgement" }, @@ -3126,8 +3127,8 @@ ], "issue": "Lead 6, third item, confirmed, with a second dead piece beside it. write-pause says 'A scope is `all`, one tenant, one read model, or one source stream type', `scopesOfUseCase` returns 'one read-model scope per maintained read model', the pipeline's step 7 names 'a read model it maintains' among the scopes that refuse, and the rebuild's `step2` passes 'the `all` scope, the scope of the tenant it iterates and the read-model scope'. Nothing closes that scope: `startGeneration` closes 'one `source:` scope per context and stream type', and `closeGate` is 'the operator's entry to `closeScope` for a tenant, a source stream type or the `all` scope'. The `allowGeneration` parameter of `gateAllows` is dead for the same reason: the only closed entries that carry a `generationId` are `source:` entries, and no batch passes a `source:` scope, so 'unless that entry's `generationId` equals `allowGeneration`' never decides anything, and the rebuild's `step2` phrase 'a closed entry covers one of those scopes and names a generation other than this one' describes an entry nothing writes. The sentence 'this generation's batches are the one background writer allowed through' is true only because the batch never asks about the sources.", "fix": "In `spec:application.write-pause`, which owns the gate, take one reading. Either remove the `read-model:` scope from `typeScopeKey`, the Contract, `scopesOfUseCase` and the rebuild's `step2`, and drop `allowGeneration`, stating that a rebuild batch is not held by the sources its own generation closed because it writes no source; or keep both and say that `closeGate` accepts a `read-model:` scope and what it is for. Align the pipeline's step 7 wording with the choice.", - "status": "open", - "note": "", + "status": "partially-fixed", + "note": "S4 on paper, 2026-10-02. `spec:application.write-pause` `typeScopeKey` and `fnScopesOfUseCase` and `spec:command.command-pipeline` step 7 drop the `read-model:` scope, and `fnGateAllows` exempts an entry only when both generation fields are present and equal. `allowGeneration` still decides nothing on the online path, because `spec:application.rebuild` `step3` reads the `all` and tenant scopes only; whether a history batch reads its source scopes is for roadmap 4c, which writes the paused path.", "slice": "S4", "detector": "convex-test" }, @@ -3312,8 +3313,8 @@ ], "issue": "Several Specs require an actor of kind `operator` and the one helper that turns an identity into an actor cannot return one. operator-operations says every operation 'runs under an actor of kind `operator` established by the parent' and its step1 is 'establish the actor from `ctx.auth` in the parent and authorize `obligations:repair` or `obligations:inspect`'. rebuild says its functions 'reach the dashboard through operator functions that authorize an operator actor'. actor-and-scope pins `establishActor` as returning '`null` ..., a `service` actor when the identity's issuer is in `serviceIssuers`, and a `human` actor otherwise'. Grants are read by kind, `.eq(\"principalKind\", actor.kind)`, so a grant with `principalKind: \"operator\"` is never matched by an actor the helper produced, and no Spec says what makes an identity an operator (an issuer, a grant, a deployment admin). The 'operator functions' that front `startGeneration`, `closeGate`, `startRestoreRun` and the other internal mutations taking `actor: actorValidator` are named and nowhere pinned. The same holds for `reviewer`, which is deferred with Layer 4; `operator` is needed from Layer 2.", "fix": "In `spec:command.actor-and-scope`, rule how an operator is established and extend the helper: for example `establishActor(ctx, config: { serviceIssuers: ReadonlySet⟨string⟩; operatorIssuers: ReadonlySet⟨string⟩ })` returning kind `operator` for an operator issuer, or a second helper `establishOperator` that requires a grant of a reserved permission and returns the same identity with kind `operator`. Mark it [extension] under E-37. Then pin one operator entry shape, a public `mutation` or `query` that establishes, authorizes and calls the internal function, and cite it from `spec:obligations.operator-operations` step1, `spec:application.rebuild`, `spec:application.write-pause` and `spec:application.restore`.", - "status": "open", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: An operator grant cannot be matched when establishActor returns only human or service, yet the maintenance and obligation contracts require operator kind. The build must show: Pure identity tests and convex-test must produce operator kind only by the ruled trust mechanism and match its grants, leaving ordinary humans and services unable to claim it; source review must align every maintenance caller with that mechanism.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: An operator grant cannot be matched when establishActor returns only human or service, yet the maintenance and obligation contracts require operator kind. The build must show: Pure identity tests and convex-test must produce operator kind only by the ruled trust mechanism and match its grants, leaving ordinary humans and services unable to claim it; source review must align every maintenance caller with that mechanism. S4 on paper, 2026-10-02. `spec:command.actor-and-scope` E-48 answers it for the deployment's operator entries: no function establishes an actor of kind `operator` from a credential, and an entry records the operator admin access states. `spec:obligations.operator-operations` still says its repair operations run under an actor of kind `operator` established by the parent, which `establishActor` cannot return; the main thread ruled that those operations keep their tenant grant, and the actor they run under waits for its ruling.", "slice": "S4", "detector": "pure unit test" }, @@ -3459,7 +3460,7 @@ "issue": "`writeAudit(ctx: MutationCtx, record: AuditRecordInput)` takes a type no Spec pins, and its two callers cannot fill the table from what they are said to have. `tableAuditRecords` requires `operationId`, `commandType`, `actor`, `subject: v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() })`, `kind` and `decision: v.string()`. The pipeline's step 10 calls it 'for a declaration that sets `audit`', and the declaration's `audit` is `{ kind: \"security\" | \"business\" }`; no bullet says which of a use case's affected streams is the one `subject` or what `decision` holds. write-pause says 'every close, resume and abort writes an audit record inside the same mutation and fails closed', through `closeScope` and `resumeScope`; a gate change has no operation ID, no command type and no stream subject, so it does not fit the required fields. One stale sentence sits beside this: `indexAuditByOperationUse` names 'the restore's receipt check' as a reader, and the restore's receipt check reads receipts and `byOperation` events, never audit records.", "fix": "In `spec:operations.baseline-operations` pin `AuditRecordInput` and say how step 10 fills it (for example `subject` from the declaration's `permission.subjectFrom(input)` or the first affected ref, `decision` from the outcome kind). For gate changes either make `operationId`, `commandType` and `subject` optional with a `scopeKey` field, or give the gate its own small audit table in `spec:application.write-pause`. Drop the restore from `indexAuditByOperationUse`.", "status": "open", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: Closing or resuming a tenant/source gate supplies a scope, reason and actor but not the required operation ID, command type and stream subject, so the mandatory audit writer cannot construct its promised input. The build must show: Compiled AuditRecordInput and convex-test must represent command and gate actions, atomically store the declared audit evidence and roll back every mandatory action on audit failure; native backend acceptance must show the same refusal and unchanged business/gate state.", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: Closing or resuming a tenant/source gate supplies a scope, reason and actor but not the required operation ID, command type and stream subject, so the mandatory audit writer cannot construct its promised input. The build must show: Compiled AuditRecordInput and convex-test must represent command and gate actions, atomically store the declared audit evidence and roll back every mandatory action on audit failure; native backend acceptance must show the same refusal and unchanged business/gate state. S4 on paper, 2026-10-02. `spec:operations.baseline-operations` `typeAuditRecordInput`, `tableOperatorAudit`, `fnWriteOperatorAudit` and `indexOperatorAuditByScopeUse` pin both shapes, and `spec:application.write-pause` `fnGetGateAudit` reads the gate's by scope. The build must show at the compiled tier that each input type is its table's document without `recordedAt`, and at convex-test one `operatorAudit` record per gate change and a refused record leaving the gate unchanged.", "slice": "S4", "detector": "TypeScript compiler" }, @@ -3493,8 +3494,8 @@ ], "issue": "Three reads name an index that cannot answer them as described. retention-and-restore `handlerShims`: 'a release removes a version only after `by_tenant_status` shows no pending or running obligation on it'; that index is `[\"tenantId\", \"status\", \"updatedAt\"]`, it carries neither `handlerKey` nor `handlerVersion` and needs a tenant, so the check is a filter over every tenant's pending and running rows, not an index read. rebuild `fnPurgeGenerationBatch` deletes 'by the `by_key` index range for that generation' and `limitPurgeBatch` says 'one index range'; `by_key` is `[\"tenantId\", \"generation\", \"key\"]` and the markers' `by_entity` leads with the tenant too, so a generation's rows are one range per tenant, and the function's args `{ generationId, fence }` and the generation row carry no purge cursor. restore `fnReceiptCheckBatch` reads '25 receipts by tenant from the receipt cursor' without naming which of the three receipt indexes it pages. The tenant iteration these need is the list `r3-architecture-tenant-list-undefined` reports as missing; the index mismatch is separate.", "fix": "In `spec:obligations.retention-and-restore` name the read: the deployment-wide `by_status_next_attempt` and `by_status_lease_expiry` ranges for `pending` and `running`, filtered on the handler key and version, bounded and paged. In `spec:application.rebuild` say that a purge batch iterates tenants like a backfill batch, add `purgeCursor: v.optional(batchCursorValidator)` to the generation row in `spec:application.generation-registry`, and reword `limitPurgeBatch` to one range per tenant. In `spec:application.restore` name `by_tenant_expiry` or `by_key` as the paged index.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-08: At HEAD, rebuild.sdp.md:110 purges \"the by_key index range for that generation\" but projection-contract.sdp.md:86 leads by tenant, and the purge has no tenant cursor, so one range cannot purge a deployment-wide generation. The issue and tenant-aware paged fix remain right; receipt and handler-version scans must also name their actual indexed ranges. The build must show: Compiled indexed queries and native backend paging must cross tenant boundaries during purge and retention and name the receipt index, reaching every intended row and resuming at the exact tenant and key after interruption.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-08: At HEAD, rebuild.sdp.md:110 purges \"the by_key index range for that generation\" but projection-contract.sdp.md:86 leads by tenant, and the purge has no tenant cursor, so one range cannot purge a deployment-wide generation. The issue and tenant-aware paged fix remain right; receipt and handler-version scans must also name their actual indexed ranges. The build must show: Compiled indexed queries and native backend paging must cross tenant boundaries during purge and retention and name the receipt index, reaching every intended row and resuming at the exact tenant and key after interruption. S4 on paper, 2026-10-02. spec:application.rebuild purgeSteps now uses tenantId and generation equalities and a saved tenant cursor, advancing only after a short deletion page. spec:obligations.retention-and-restore handlerShims and spec:application.restore fnReceiptCheckBatch still need their bounded index reads from those lanes.", "slice": "S4", "detector": "TypeScript compiler" }, @@ -3613,8 +3614,8 @@ ], "issue": "rebuild `migrationsBinding` pins `const migrations = new Migrations⟨DataModel⟩(components.migrations, { internalMutation })` with `migrations.define({ table: \"orderSummaries\", batchSize: 100, customRange: (q) => q.withIndex(\"by_key\", (i) => i.eq(\"tenantId\", tenantId).eq(\"generation\", generation)), migrateOne })` as the runner 'where the batch's source is a parent table, such as purging a generation or re-projecting an existing generation's rows'. Against the component's README and source (github.com/get-convex/migrations, 0.3.6, README.md and src/client/index.ts, read 2026-10-01) this cannot run. (1) 'Note: to use customRange, you must provide your schema to Migrations'; the client throws 'You must provide your schema to use a custom range.' when `options.schema` is absent, and the pinned constructor passes no schema. (2) `define` is a static top-level export whose run arguments are `fn`, `cursor`, `batchSize`, `dryRun`, `reset` and `next`; `customRange` receives only the query, so `tenantId` and `generation` in the pinned closure are free variables with no way to bind them per run. A range that differs per generation and per tenant cannot be expressed. (3) The component tracks state per migration name and 'if a migration had already completed, it will skip it' in a series, so one defined migration cannot serve generation after generation without `reset`. Meanwhile the same Spec pins a complete hand-written runner for the same job, `purgeGenerationBatch` with a generation ID and a fence, and generation-registry adds `migrationsState` to reconcile two cursors. An implementer cannot tell which runner purges, and the one the doc names (D9, F17) does not work as written. Two statements around it are also stale: F17 and rebuild's assumption say the batch size is 'not stated', but the README has a 'Custom batch size' section ('The component will fetch your data in batches of 100 ... you can specify it', in `define` and per `runOne`); and rebuild leaves 'whether a migration can iterate a table inside a component' to Probe 6 while journal `driverNotMigrationsComponent` already states it cannot, which follows from 'Components expose an explicit API, not direct database table access' (docs.convex.dev/components/understanding, read 2026-10-01) and from `define` taking a `table` of the app's own data model.", "fix": "On `spec:application.rebuild` (owner of E-40) rule one way. Either drop `migrationsBinding` and `migrationsState` and state that every Layer 2 batch, backfill, verify and purge, is the self-scheduled internal mutation with the generation row as its checkpoint, with `@convex-dev/migrations` kept for ordinary schema migrations of parent tables only, which is what the Specs already do everywhere else; or keep it for a fixed range and pin it correctly: `new Migrations(components.migrations, { schema, internalMutation })`, a `customRange` with no run-time variables, and `runOne(ctx, ref, { reset: true, batchSize })` per use. Record the choice against D9's wording in the E-40 question. Update `spec:facts.f17-migrations-fits-generation-backfill`: batch size is documented in the README, a component's table cannot be iterated from the app, so what stays assumed is only the race of a batch against a live command. Narrow Probe 6 in `spec:facts.probe-plan` the same way.", - "status": "open", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: A migrations runner with a generation- and tenant-dependent range has no installed package or complete per-run binding, and two owners can checkpoint the same batch. The build must show: Compiled code must bind the schema, fixed or ruled per-run range and restart/reset policy; native backend Probe 6 must show one checkpoint owner and resumable backfill or purge, without installing a generic migrations runner before the chosen path needs it.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: A migrations runner with a generation- and tenant-dependent range has no installed package or complete per-run binding, and two owners can checkpoint the same batch. The build must show: Compiled code must bind the schema, fixed or ruled per-run range and restart/reset policy; native backend Probe 6 must show one checkpoint owner and resumable backfill or purge, without installing a generic migrations runner before the chosen path needs it. S4 on paper, 2026-10-02. spec:application.rebuild purgeDriver and spec:application.generation-registry tableGenerationProgress remove migrationsBinding, migrationsState and their unusable wiring. spec:facts.f17-migrations-fits-generation-backfill, spec:facts.probe-plan, spec:decisions.d09-rebuild-online-by-default and spec:context.journal still need the evidence and probe alignment in their owning lanes.", "slice": "S4", "detector": "TypeScript compiler" }, @@ -3629,7 +3630,7 @@ "issue": "rebuild says of a backfill batch, which is a scheduled internal mutation: 'a batch that keeps losing the optimistic-concurrency race fails after Convex's own retries with its documents-changed error, and a developer error, that failure or a timeout ends the batch', and `occRace` repeats 'a batch that keeps losing fails after Convex's internal retries with its documents-changed error, which is the stall that `resumeGeneration` answers'. The bounded retry and that error are documented for a mutation a caller invokes (docs.convex.dev/error, 'some of the invocations will eventually throw this error', read 2026-10-01). For a scheduled mutation the scheduling page says: 'Once scheduled, mutations are guaranteed to be executed exactly once. Convex will automatically retry any internal Convex errors, and only fail on developer errors' (docs.convex.dev/scheduling/scheduled-functions, read 2026-10-01), and the open-source scheduler shows what that means for a conflict: in `handle_mutation` an OCC error on commit runs `backoff.fail(); wait; continue` with no retry cap (get-convex/convex-backend, crates/application/src/scheduled_jobs/mod.rs, `SCHEDULER_OCC_MAX_BACKOFF` 60 seconds, read 2026-10-01). So a contended batch is never marked failed and never ends its chain. It stays pending and is retried with growing backoff, the generation row's `lastError` stays empty, and the stall shows only as an `updatedAt` that stops moving. The operator signal the Spec describes does not occur. The mechanism survives: `resumeGeneration` bumps the fence and the old batch, when it finally commits, returns on the stale fence.", "fix": "In the flow bullet and in `occRace` replace 'fails after Convex's own retries with its documents-changed error' with what Convex does for a scheduled mutation: it is retried with backoff until it commits, so a stalled generation is one whose `updatedAt` has not advanced while its batch is still pending. State that as the stall signal in the E-40 question and in `resumeGeneration`, and keep the documents-changed error for mutations a caller invokes, as persistence-adapter `versionConflictVersusOccRetry` already has it. Mark the unbounded retry as read from the backend source, not from a docs page, and add it to Probe 6.", "status": "open", - "note": "", + "note": "S4 on paper, 2026-10-02. spec:application.rebuild assumption on scheduled mutation reruns and stallSignal replace retry exhaustion with nonadvancing progress. Unbounded retry remains an assumption; Probe 6 and the backend-source attribution required by the finding remain for the probe lane.", "slice": "S4", "detector": "judgement" }, @@ -3780,7 +3781,7 @@ "issue": "queries `fnList` caps `numItems` and `maximumRowsRead` at the same number, `limitListPage`, and `fnHistory` does the same with `limitHistoryPage`. In the helper the row cap is tested before the item target: `if (maxRowsToRead !== undefined && indexKeys.length >= maxRowsToRead) hitLimit = true`, then `if (hitLimit) { pageStatus = \"SplitRequired\"; splitCursor = ... }` (convex-helpers 0.1.124, packages/convex-helpers/server/stream.ts `paginate`, read 2026-10-01). So a first run that asks for `limitListPage` items or more and finds that many rows always comes back `SplitRequired`, although nothing grew. The `convex-helpers/react` hook treats that status as an incomplete page: 'it means the server was not able to fetch the full page. So we stop results before the incomplete page and return LoadingMore while the page is splitting' (react.ts, read 2026-10-01), so every full first page costs a second round trip and two subscriptions in place of one. The rebuild's enumeration and the restore's check batches get the same status on every full page; they are correct only if they ignore it, which no Spec says. The Spec records the split as a cost of a page that fills 'with live inserts', not of an ordinary full page.", "fix": "In `spec:context.queries` keep the byte bound as it is and set the row cap one step above the item cap, `maximumRowsRead: 2 * limitListPage` with `limitListPage` halved in `limitListPage` and `limitHistoryPage` so the 8 MiB and 9 MiB arithmetic is unchanged, or state plainly that a page of exactly the cap is always split and count the extra query in the risk bullet. State in `spec:application.rebuild` `fnEnumerateCall` and `spec:application.restore` `fnStreamCheckBatch` that a non-reactive caller reads `page`, `isDone` and `continueCursor` and ignores `pageStatus`, because the rows up to `continueCursor` are complete.", "status": "partially-fixed", - "note": "S1, 2026-10-01. Slice S1 built no list or history query (ruling R2). S2, which builds lists, must set the row cap against the item cap in `spec:context.queries`, or state that a full page is always split, and show in a native test whether a first page of exactly the cap comes back `SplitRequired`. S2, 2026-10-02, fold-in after the build. A list caps `numItems` at its item cap and passes a row cap of twice it, so a full page is never split on its own (`spec:context.queries`); shown at the native tier on the fixture composition by `tests/native/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split.test.ts` and its equal-cap sibling, and on the production composition by `tests/native/read-models.list-pages-by-cursor.test.ts`. The rebuild's page bounds now carry the new numbers. Left for S4: `fnEnumerateCall` in `spec:application.rebuild` does not say that a caller that does not subscribe ignores `pageStatus`; the same for `fnStreamCheckBatch` in `spec:application.restore`, for S5.", + "note": "S1, 2026-10-01. Slice S1 built no list or history query (ruling R2). S2, which builds lists, must set the row cap against the item cap in `spec:context.queries`, or state that a full page is always split, and show in a native test whether a first page of exactly the cap comes back `SplitRequired`. S2, 2026-10-02, fold-in after the build. A list caps `numItems` at its item cap and passes a row cap of twice it, so a full page is never split on its own (`spec:context.queries`); shown at the native tier on the fixture composition by `tests/native/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split.test.ts` and its equal-cap sibling, and on the production composition by `tests/native/read-models.list-pages-by-cursor.test.ts`. The rebuild's page bounds now carry the new numbers. Left for S4: `fnEnumerateCall` in `spec:application.rebuild` does not say that a caller that does not subscribe ignores `pageStatus`; the same for `fnStreamCheckBatch` in `spec:application.restore`, for S5. S4 on paper, 2026-10-02. spec:application.rebuild Design enumeration paragraph explicitly reads page, isDone and continueCursor and ignores split status. The existing note already records the spec:context.queries fix and its native tests. spec:application.restore fnStreamCheckBatch still omits the nonreactive split-status rule, for the restore lane.", "slice": "S4", "detector": "convex-test" }, @@ -3818,8 +3819,8 @@ ], "issue": "Lead 5 holds as a design, with three gaps. write-pause `fnGateAllows` 'reads `process.env.MAINTENANCE_MODE` and then the gate document once' in every command and batch, claim-call-settle `claimStep2` tests `process.env.OBLIGATIONS_DISPATCH === \"off\"`, and restore says 'The operator sets `MAINTENANCE_MODE` to `restore` on the target deployment, so the writer check refuses every write with `writePaused` from now on'. Reading `process.env` inside a mutation handler is supported ('You can access environment variables in Convex functions using `process.env.KEY`', and `process.env` is listed for the default runtime; docs.convex.dev/production/environment-variables and docs.convex.dev/functions/runtimes, read 2026-10-01). (1) No page says in one sentence that a changed value is seen by the next function run without a deploy. The page implies it: exports conditioned on a variable 'will throw an error at runtime, if the DEBUG environment variable changes between deployment and calling the function', 'environment variables used in cron definitions will only be reevaluated on deployment', and a variable passed by reference to a component 'will always match ... even when it changes between deploys'. The open-source backend shows the mechanism: `op_environment_variables_get` reads through the function's transaction (`PreloadedEnvVars::get(tx, name)`, crates/udf/src/environment.rs and crates/isolate/src/environment/udf/phase.rs, read 2026-10-01), so the value is current per run and the read joins the read set. The corpus relies on this and cites only F12 for it; it is an unmarked assumption with no probe. No query reads either variable, so no subscription depends on a change. (2) Both switches are compared as raw strings. A value such as `Off` or `false` leaves `OBLIGATIONS_DISPATCH === \"off\"` false, so provider calls go out during a restore; `Restore` leaves the door open. Convex can refuse such a value: declared variables give 'validation at deploy time' and help 'prevent removing required environment variables or setting them to invalid values from the dashboard or `npx convex env [remove|set]`', with `v.union(v.literal(...))` validators and a typed `env` import (same page, 'Declaring environment variables'). (3) The L2-8 and L3-7 examples flip both switches during a run, and the local backend page lists 'No way to set environment variable values from within tests' (docs.convex.dev/testing/convex-backend, read 2026-10-01); the harness has to run `npx convex env set` against the backend, which the guide itself does for `IS_TEST`, and no Spec says so.", "fix": "On `spec:application.write-pause` (owner of the door) declare both switches in the parent's `convex.config.ts`: `defineApp({ env: { MAINTENANCE_MODE: v.optional(v.literal(\"restore\")), OBLIGATIONS_DISPATCH: v.optional(v.union(v.literal(\"on\"), v.literal(\"off\"))) } })`, read them through the typed `env` of `_generated/server` inside the handler, and state what an unset value means for each. Record as an assumption on write-pause, restore and retention-and-restore that a change is seen by the next function run without a deploy, cite the page and the source file, and add the check to Probe 7 in `spec:facts.probe-plan`: set the variable, run a mutation, read the refusal. State on `spec:platform.acceptance-contract` that a native or end-to-end test changes a variable through `npx convex env set` from the harness, never from inside a function.", - "status": "open", - "note": "", + "status": "partially-fixed", + "note": "S4 on paper, 2026-10-02. spec:application.write-pause environment assumption and fnGateAllows mark the next-run behavior as unprobed and specify exact restore and unset/other values. Typed env declarations, source attribution and native backend next-run verification remain; spec:application.restore and spec:obligations.retention-and-restore must share the switch rules.", "slice": "S4", "detector": "native local-backend test" }, @@ -3907,8 +3908,8 @@ ], "issue": "Rounds 1 and 2 moved the adapter, the list page, the backfill, the baseline batch and the restore's stream check from document counts to bytes. Six batches still state a bound in documents only, and at the corpus's own row ceilings each crosses 'Data read 16 MiB' or 'Data written 16 MiB' per transaction (docs.convex.dev/production/state/limits#transactions, read 2026-10-01), against rubric item 29. (1) `receipts.sweep`: 'at most 1,000 receipts per run for one tenant ... so one run stays far under the 16,000 documents written per transaction' (idempotency `limitSweepBatch`, receipt-table `fnSweep`). receipt-table `limitRowSize` lets a row reach 400 KiB; the receipt of the experiment's own 100-line `PlaceOrder` holds 101 affected refs and 101 stream versions, about 30 KiB, so 1,000 such rows read about 29 MiB. A mutation is deterministic, so a sweep that crosses the ceiling fails at the same rows on every run and expired receipts are never removed. (2) restore `fnReceiptCheckBatch`: '25 receipts per receipt batch, each one component call', where the call is `byOperation`, which 'ranges `by_operation` and collects' every event of the operation; 25 receipts at 400 KiB are 9.8 MiB before any event, and one operation may have appended up to 8 MiB of events (adapter `limitDocumentsWrittenPerCall`), so the batch has no byte bound at all. A failed check batch leaves the restore closed. (3) rebuild `limitPurgeBatch`: '500 rows deleted per purge batch, one index range, under 1 second'; at the 64 KiB `rowBudgetBytes` ceiling 500 rows read 31 MiB, while the backfill beside it is derived from the same budget. (4) sweeper `limitSweepBatch`: '100 candidates per scan and 100 expired leases per scan as the document bound, so one run reads at most about 400 documents', with an inline payload of up to 64 KiB on each row (record-contract `limitInlinePayload`) and each candidate read again inside `recoverOne`, up to about 19 MiB under the shared limits. (5) retention `limitRetentionBatch`: '500 deletions and 500 compactions per run ... under the 16,000 writes per transaction', up to 32 MiB read for the deletions alone. (6) fan-out `limitFanOutPage`: '500 derived obligations per fan-out attempt ... under the 1000 schedules per mutation and the write ceilings', which at a 64 KiB payload writes 31 MiB. The first three are Layer 1 and 2 and sit in the first experiment's path.", "fix": "Derive sweep, receipt-check, purge, retention and fan-out bounds from complete measured row and event costs, including rereads, metadata and schedules. Page operation history with a resumable byte and row bound and persist caller progress across pages. At 64 KiB, floor(4 MiB / 64 KiB) is 64, not 62, and payload size alone is not row cost. If transaction metrics select headroom, await the pinned API and read metrics.bytesRead.remaining from the resolved result; each metric has { used, remaining } fields. Preserve provisional numeric defaults until native backend boundary cases prove progress within shared limits.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-08: At HEAD, rebuild.sdp.md:124 deletes \"500 rows\" and restore.sdp.md:100 checks \"25 receipts\" with unbounded byOperation calls, so 64 KiB purge rows alone read 31.25 MiB and one migration operation can exceed a transaction. The issue holds, but STATE.md Leads 2 invalidates the fix's 62-row arithmetic, payload-as-row estimate and synchronous bytesRead access; derive complete row costs, page operation history and use any metric through its awaited { used, remaining } shape. the platform review of 2026-10-02, CORE-07: At HEAD, context/queries.sdp.md:81 says byOperation \"collects\" while journal.sdp.md:106 gives every baseline in a migration one operationId, so 100 baselines near 256 KiB exceed 25 MiB under one ID despite safe individual batches. Bound operation-history pages and the receipt-check caller by bytes and rows; a resumable answer must let the next batch advance rather than retry the same refused operation. The build must show: Native backend cases must purge near-limit read-model rows across tenants and check receipts for a multi-batch migration whose total baseline payload exceeds one transaction, with bounded operation-history pages reaching every event once. Sweep, retention and fan-out boundary cases must advance the exact saved position without repeated limit failures, counting complete row cost, rereads and schedules.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-08: At HEAD, rebuild.sdp.md:124 deletes \"500 rows\" and restore.sdp.md:100 checks \"25 receipts\" with unbounded byOperation calls, so 64 KiB purge rows alone read 31.25 MiB and one migration operation can exceed a transaction. The issue holds, but STATE.md Leads 2 invalidates the fix's 62-row arithmetic, payload-as-row estimate and synchronous bytesRead access; derive complete row costs, page operation history and use any metric through its awaited { used, remaining } shape. the platform review of 2026-10-02, CORE-07: At HEAD, context/queries.sdp.md:81 says byOperation \"collects\" while journal.sdp.md:106 gives every baseline in a migration one operationId, so 100 baselines near 256 KiB exceed 25 MiB under one ID despite safe individual batches. Bound operation-history pages and the receipt-check caller by bytes and rows; a resumable answer must let the next batch advance rather than retry the same refused operation. The build must show: Native backend cases must purge near-limit read-model rows across tenants and check receipts for a multi-batch migration whose total baseline payload exceeds one transaction, with bounded operation-history pages reaching every event once. Sweep, retention and fan-out boundary cases must advance the exact saved position without repeated limit failures, counting complete row cost, rereads and schedules. S4 on paper, 2026-10-02. `spec:application.rebuild` `limitPurgeBatch` derives the purge count from `rowBudgetBytes` and states its read with each row's system fields, and `purgeSteps` iterates tenants. The native backend must show a purge near its bound; receipts, restore, obligations, retention and fan-out stay.", "slice": "S4", "detector": "native local-backend test" }, @@ -3941,7 +3942,7 @@ "issue": "During an online rebuild every live command and every backfill batch meet on one document. The pipeline's step 9 calls `generationsToWrite`, which makes 'at most two indexed reads' of `by_read_model_state` and so reads the row of the generation in `building`, `verifying` or `verified`. The same row is the batch's checkpoint: 'The generation row carries the backfill cursor, the verify cursor, the fence and the counts, and a batch writes them in the same mutation as its rows' (generation-registry Contract), and `step5` 'patches the generation row with the new cursor, the counts and `updatedAt`' on every batch, with batches chained by `ctx.scheduler.runAfter(0, ...)`. Convex decides conflicts on the read set: 'At the end of the transaction, the writes all commit if every version in the read set is still the latest version of that record' (docs.convex.dev/database/advanced/occ, read 2026-10-01), and a write to a document inside a range a mutation read conflicts whatever field changed ('any change to the \"tasks\" table will conflict with the `writeCount` mutation', docs.convex.dev/error#1, read 2026-10-01). So each batch commit invalidates every command on that read model that is in flight, on any tenant, and the command re-runs. A caller's mutation gets a bounded number of retries, after which 'some of the invocations will eventually throw this error' (same page). With batches committing back to back, a short command re-runs now and then and a long one, the 100-line `PlaceOrder`, overlaps a batch commit on most tries and can exhaust its retries, for the whole length of the rebuild. The corpus says the opposite: 'The common rebuild costs no downtime' (rebuild value), 'The registry is data that changes only during a rebuild, so it is not an application-wide counter in the write path' (generation-registry), and it records the one-time re-run for the gate ('a gate change makes every concurrent command re-run once') but not this repeated one. Probe 6 races a batch against a command on a read-model row, not on the registry row. How often a command loses is unmeasured; that it conflicts on every overlapping batch commit is documented behavior.", "fix": "In `spec:application.generation-registry` split the row: keep on `generations` only what a command and a query read, `readModel`, `generation`, `projectionVersion`, `state`, `pauseRequired` and `fence`, which change at start, switch, resume, rollback and abort; move `cursor`, `verifyCursor`, `batchSize`, `batchesDone`, `rowsWritten`, `rowsSkipped`, `misses`, `lastError` and `updatedAt` to a `generationProgress` table keyed by generation ID that only batches and operators read. A batch then checks state and fence on the registry row, which it reads and does not write, and checkpoints on the progress row, which no command reads. Update rebuild's steps 1 and 5, `resumeChain`, the restore's inventory and `finishRestoreRun` to the two rows. Add the registry-row case to Probe 6 in `spec:facts.probe-plan` and to the first experiment's contention run: `PlaceOrder` at the maximum while a backfill runs, counting retries and exhaustions.", "status": "open", - "note": "the platform review of 2026-10-02, APPL-07: At HEAD, generation-registry.sdp.md:47 makes the registry row \"the checkpoint\" and rebuild.sdp.md:115 patches it every batch, so each overlapping command selecting that row can conflict even on unrelated streams. The issue and separate progress-row fix remain right; the frequency and throughput consequence need native backend measurement, not convex-test. The build must show: Source review must show disjoint command-selection and batch-checkpoint write sets, and a native backend comparison must run maximum-size and unrelated-stream commands during backfill against a no-rebuild control, reporting retries and exhaustions rather than inferring throughput from convex-test.", + "note": "the platform review of 2026-10-02, APPL-07: At HEAD, generation-registry.sdp.md:47 makes the registry row \"the checkpoint\" and rebuild.sdp.md:115 patches it every batch, so each overlapping command selecting that row can conflict even on unrelated streams. The issue and separate progress-row fix remain right; the frequency and throughput consequence need native backend measurement, not convex-test. The build must show: Source review must show disjoint command-selection and batch-checkpoint write sets, and a native backend comparison must run maximum-size and unrelated-stream commands during backfill against a no-rebuild control, reporting retries and exhaustions rather than inferring throughput from convex-test. S4 on paper, 2026-10-02. `spec:application.generation-registry` `tableGenerationProgress`, `transitionCost` and the checkpoint question under E-40 move the cursor, the batch size and the counts to a progress row no command or query reads, and a batch patches the generation row only in the mutation that ends its pass. The build must show, in a contention run on the native backend against a run with no rebuild, that no command fails on a registry conflict while batches checkpoint.", "slice": "S4", "detector": "measurement or contention run" }, @@ -4594,10 +4595,11 @@ ], "issue": "No example or verification bullet shows that a read-model row deleted by a `null` projection comes back in the active generation on the next live-updated write, the case `spec:application.projection-contract` states in its rule that a live write to the active generation always lands.", "fix": "Add to `spec:application.projection-contract` Verification a bullet that a test deletes a row through a `null` projection and asserts the next live-updated write to the active generation answers `inserted`, or add that step to the L2-5 example.", - "status": "open", - "note": "Remainder of r3-completeness-apply-projection-cannot-tell-active-from-building, left after its fix when a lens read checked the fix against the Spec text.", + "status": "fixed", + "note": "Remainder of r3-completeness-apply-projection-cannot-tell-active-from-building, left after its fix when a lens read checked the fix against the Spec text. S4 on paper, 2026-10-02. spec:application.projection-contract final Verification rule explicitly requires a null deletion followed by live-updated inserted in active and verifying and skipped-missing in building, closing the requested missing assertion.", "slice": "S4", - "detector": "lens read of a flagged fix" + "detector": "lens read of a flagged fix", + "unreviewed": true }, { "round": "PR1", diff --git a/design/specs/application.pack.sdp.md b/design/specs/application.pack.sdp.md index 589e1c1..ebf6e49 100644 --- a/design/specs/application.pack.sdp.md +++ b/design/specs/application.pack.sdp.md @@ -46,4 +46,4 @@ modelRefs: Package D. Layer 2: parent use cases, read models and the projection contract, online rebuild with the generation registry and the write pause, restore, baseline operations, the cost-target constraints and the first experiment, with the Layer 2 scenarios and the all-layers audit scenario. -Membership order follows the plan's inventory with each parent's examples directly after it: parent use cases, read models, the projection contract, rebuild with its registry and gate, restore, the first experiment with the example domain, baseline operations, then the eight constraints. Thirty-seven Specs: nine application Specs, one operations Spec, eight constraints and nineteen examples. Three examples are siblings the plan's table did not list, `second-context-throws`, `committed-state-read-through-query` and `write-pause-rebuild-abort`, added because each doc row enumerates two cases. Five examples verify no doc row: `first-activation-makes-read-model-writable` and `command-without-generation-fails` verify the generation registry's first activation and its failure under E-8, `query-refused-before-disclosure` and `query-refused-without-grant` verify Law 5 for a parent query, and `list-pages-by-cursor` verifies a parent list under E-24. +Membership order follows the plan's inventory with each parent's examples directly after it: parent use cases, read models, the projection contract, rebuild with its registry and gate, restore, the first experiment with the example domain, baseline operations, then the eight constraints. Thirty-seven Specs: nine application Specs, one operations Spec, eight constraints and nineteen examples. Three examples are siblings the plan's table did not list, `second-context-throws`, `committed-state-read-through-query` and `write-pause-rebuild-abort`, added because each doc row enumerates two cases. Five examples verify no doc row: `first-activation-makes-read-model-writable` and `command-without-generation-fails` verify the generation registry's first rebuild and the failure of a command before it under E-8, `query-refused-before-disclosure` and `query-refused-without-grant` verify Law 5 for a parent query, and `list-pages-by-cursor` verifies a parent list under E-24. diff --git a/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md b/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md index e82b5a6..3a67e5e 100644 --- a/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md +++ b/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md @@ -29,4 +29,4 @@ And a second start while generation 1 is being built is {second: "refused"} - The test places three orders through a command that declares no read model, or writes them before the read model is declared, then runs `internal.rebuild.startGeneration` for the order summary with admin access and a stated operator, and asserts one row in `generations`, generation 1, `building`, with the order summary's latest projection version and the operator as `startedBy`. - The test runs `startGeneration` again before the first generation is switched and asserts that its error names the read model, generation 1 and its state, and that the table still holds one row. - The test waits until the row is `verified`, asserts that a query over the read model still fails for want of an active generation, runs `switchGeneration`, and asserts that the row is `active`, that no row is `retired`, and that the list query returns the three orders. -- The test runs the same sequence on a backend with no subject and asserts that the progress row shows one backfill batch and one verify batch before `verified`. +- The test runs the same sequence on a backend with one tenant and no subject and asserts that the progress row's `batchesDone` is 2, one backfill batch and one verify batch, before `verified`. diff --git a/design/specs/application/generation-registry.sdp.md b/design/specs/application/generation-registry.sdp.md index 7e5217a..3e1d71a 100644 --- a/design/specs/application/generation-registry.sdp.md +++ b/design/specs/application/generation-registry.sdp.md @@ -44,11 +44,11 @@ The registry is two parent tables. The `generations` table records every generat - At most one generation per read model is `active`, and at most one is in `building`, `verifying` or `verified`; these three states are the in-flight states (E-8, D9) - The allowed transitions are none to building by `startGeneration`, building to verifying by the backfill batch that ends the enumeration, verifying to verified by the verify batch that ends its pass, verified to active by `switchGeneration`, active to retired by the same `switchGeneration`, retired to verifying by `rollbackGeneration`, building, verifying or verified to aborted by `abortGeneration`, and retired or aborted to purged by the purge batch that ends its pass; no other transition exists, and none leads from nothing to active (E-8, D9) - An operator entry of `spec:application.rebuild` makes the transitions named after it, a scheduled batch makes the other three, and nothing else writes a generation row (E-8, E-48, D19) -- [extension] A read model's first generation is generation 1 in `building`, started by `startGeneration` like any other, and the read model has an active generation only after that generation is verified and switched; on a deployment with no subject the build is one backfill batch and one verify batch (E-8, D9) +- [extension] A read model's first generation is generation 1 in `building`, started by `startGeneration` like any other, and the read model has an active generation only after that generation is verified and switched; on a deployment with no subject the build is one backfill batch and one verify batch for each tenant of the tenant list, and one of each when the list is empty (E-8, D9) - A command that maintains a read model writes the active generation and the in-flight generation if one exists; a retired generation is kept unchanged until it is purged and never written, and a rollback reopens it as `verifying` so that live commands write it again while its verify pass repairs it (D9, E-8) - [extension] Each generation is written with the projection whose version its row records, on the live path and by its batches alike, so the rows of one generation are one truth of one projection version (E-9, E-8, D8) - A query reads the active generation and nothing else; a query over a read model that has no active generation throws a plain error that names the read model, a technical failure, and never answers an empty page (D9, E-8, Law 6) -- [extension] The generation row carries the state, the projection version, the fence and who started and last changed it, and nothing a batch writes on its healthy path; the progress row carries the cursor, the batch size and the counts, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) +- [extension] The generation row carries the state, the projection version, the fence and who started and last changed it, and nothing a batch writes on its healthy path except the state that the batch ending a pass sets; the progress row carries the cursor, the batch size and the counts, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) - [extension] A generation spans every tenant of the deployment and the registry row names none; the tenant is named on every read-model row and marker, on every context call a batch makes, and in the batch cursor, so no read or write of tenant data runs with an absent tenant as a wildcard, and no generation can be switched for one tenant alone (E-8, D11, Law 11) - An aggregate read model keeps one marker per entity per generation with the contribution last recorded and the source versions it was computed from, and every aggregate write reads and writes the marker in the same mutation as the aggregate row; the marker's `sourceVersions` are compared like a row's, so a DTO older than the marker never adjusts the sum (D9, E-8, E-9) - [extension] A generation whose progress row is in the phase `backfill`, `verify` or `purge` has at most one scheduled batch driving it, identified by the row's fence; a restore brings the rows back at their checkpoint and drops that batch with every pending scheduled function, so the generation is stranded until `resumeChain` bumps the fence and reschedules from the saved cursor, which the restore procedure's pending-work inventory and accepted branch drive for every such row (E-42, E-8, F12, D19) @@ -69,7 +69,7 @@ Tables live in the parent's `schema.ts`, spread from the library's `readModelTab - indexGenerationsByReadModelStateUse: `generationsToWrite` in every command that maintains the read model, `activeGeneration` in every query over it, and the in-flight check of `startGeneration`, `switchGeneration` and `rollbackGeneration` (D9, E-8) - validatorBatchCursor: `const batchCursorValidator = v.object({ tenantId: v.string(), pageCursor: v.union(v.string(), v.null()), streamId: v.optional(v.string()), eventCursor: v.optional(v.union(v.string(), v.null())) })` the position of a pass, the tenant being iterated and the `paginator` cursor within it, `null` at the start of a tenant; `streamId` and `eventCursor` are set only by the batches of a history view (E-8, E-40, Law 11) - validatorProgressPhase: `const progressPhaseValidator = v.union(v.literal("backfill"), v.literal("verify"), v.literal("purge"), v.literal("idle"))` naming the pass a batch chain is driving, `idle` when none is (E-40) -- tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), phase: progressPhaseValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position of the pass `phase` names and `null` before that pass has read its first tenant; `batchesDone` counts every committed batch of the generation, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted (E-40, D9, F13) +- tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), phase: progressPhaseValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position of the pass `phase` names and `null` before that pass has read its first tenant; `batchesDone` counts every batch of the generation that reached its checkpoint, the batch that ends a pass included and a parked batch not, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted (E-40, D9, F13) - indexProgressByGeneration: `.index("by_generation", ["generationId"])` (E-40) - indexProgressByGenerationUse: the one read a batch and an operator entry make to find a generation's progress row; no command and no query over a read model reads the table (E-40, F1) - tableProjectionMarkers: `projectionMarkers: defineTable({ tenantId: v.string(), readModel: v.string(), generation: v.number(), entityKey: v.string(), aggregateKey: v.string(), contribution: v.number(), sourceVersions: v.array(streamVersionValidator) })` (E-8, D9, Law 11) @@ -81,7 +81,7 @@ Tables live in the parent's `schema.ts`, spread from the library's `readModelTab - fnActiveGeneration: `activeGeneration(ctx: RegistryReader, readModel: string): Promise` making one indexed read and answering `undefined` when the read model has no active generation, which a query over the read model then turns into the plain error `"The read model " + readModel + " has no active generation"`; the answer is the same for every tenant, and the query narrows the rows it reads by its own `tenantId` argument (E-8, D9, Law 11) - errorNoGenerationToWrite: the pipeline's step 9 throws `commandType + " writes the read model " + readModel + ", which has no generation to write"` when `generationsToWrite` answers an empty list, which is the case before `startGeneration` and after the only generation was aborted (E-8, D8, Law 6) - markerVersionCompare: `applyAggregate` treats the marker's `sourceVersions` as a row's: a marker whose versions are not older than the input's makes the write `skipped-newer` and leaves the sum alone, so a stale DTO on either path never adjusts an aggregate twice or backwards (E-9, E-8, D9, F1) -- transitionCost: a transition patches one generation row, or two in the one write of a switch that retires an active generation, a single mutation, and every command in flight on the read model that read the registry before it re-runs once; a batch writes a generation row only in the mutation that ends its pass, so a build costs the commands three such re-runs, at the start, at the end of the backfill and at the end of the verify pass, plus one per operator entry (E-40, F1, D9) +- transitionCost: a transition patches one generation row, or two in the one write of a switch that retires an active generation, a single mutation, and every command in flight on the read model that read the registry before it re-runs once; a batch writes a generation row only in the mutation that ends its pass, so a build costs the commands one such re-run at the end of the backfill and one at the end of the verify pass, plus one per operator entry, `startGeneration` and `switchGeneration` among them (E-40, F1, D9) - limitRegistryReadsPerCommand: at most 2 indexed reads per read model per command, because the pipeline's step 9 calls `generationsToWrite` once per read model name however many bindings name it, and one per query (E-8, F13) - limitRollbackPeriod: 7 days by default, 604,800,000 ms; `switchGeneration` takes an override between 0 and 2,592,000,000 ms, and `rollbackGeneration` is refused once `retireAfter` has passed because the rows may be purged (E-8, D9) - limitMarkerCount: one marker per entity per generation; a read model with N entities costs N markers per generation (E-8, F13) diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 1bea44d..e8e564c 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -103,24 +103,24 @@ Each batch is one scheduled internal mutation in the parent. The rebuild schedul The enumeration is the context's `list` query with `includeDeleted`, the one read of context data per batch: one call returns a page of up to `batchSize` DTOs and a continuation cursor, built inside the component by `paginator` from `convex-helpers` over the `by_identity` index that includes deleted subjects, and capped by the query at its byte-derived page bound. A batch reads `page`, `isDone` and `continueCursor` and ignores any split status, because the rows up to `continueCursor` are complete. A DTO carries the stream version the projection compares, as the queries contract's `toDto(state, meta)` maps it onto `version`. Deleted subjects reach the projection as DTOs the projection maps to `null`, so their rows are deleted rather than left behind. -- transactionBoundary: each batch is one scheduled internal mutation that commits its rows, its checkpoint and the scheduling of the next batch together; every operator entry is one mutation (D9, F8, F9) +- transactionBoundary: each batch is one scheduled internal mutation that commits its rows, its checkpoint and the scheduling of the next batch together; every operator entry that changes anything is one mutation, and `getGenerations` is one query (D9, F8, F9) - convexSurface: `startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` as the operator entries, `backfillBatch`, `verifyBatch` and `purgeBatch` as the scheduled batches, all internal mutations, and `getGenerations` as an internal query, in the composition's `rebuild` module; the gate entries are in the write-pause contract (D9, E-8, E-48) - typeRebuildTarget: `type RebuildTarget = { readModel: ReadModel; source: SourceRef; list: FunctionReference<"query", "internal", ListArgs, PaginationResult> }` one per-entity read model the deployment can rebuild, its one source stream type, and the reference to that stream type's `list` query in its context component, such as `components.orders.queries.order.list` (E-8, E-9, D10) - typeRebuildConfig: `type RebuildConfig = { targets: readonly RebuildTarget[]; refs: { backfillBatch: BatchRef; verifyBatch: BatchRef; purgeBatch: BatchRef } }` where `type BatchRef = FunctionReference<"mutation", "internal", { generationId: Id<"generations">; fence: number }, null>` and `refs` are the composition's own registered batches, which a batch schedules by reference (E-40, F8) -- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; then it inserts the generation row with the next number, `state: "building"`, `pauseRequired: false`, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `phase: "backfill"`, the batch size, `cursor: null` and every count 0, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48) +- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; then it inserts the generation row with the next number, `state: "building"`, `pauseRequired: false`, since a `RebuildTarget` is a per-entity read model and the paused start of a history view is given only in outline, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `phase: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48) - fnBatchSizeFor: `batchSizeFor(readModel: ReadModel, requested?: number): number` throwing `"The batch size must be a whole number of at least 1, and " + requested + " is not"` unless `requested` is absent or an integer of at least 1, and returning `min(requested ?? rebuildBatchCeiling(readModel), rebuildBatchCeiling(readModel))`, so a batch size of 0, a negative one and a fraction are refused and a larger one is lowered to the ceiling (E-40, F13) - fnBackfillBatch: `export const backfillBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9, E-40) - fnVerifyBatch: `export const verifyBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9) - fnPurgeBatch: `export const purgeBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D19, E-40) - fnEnumerateCall: `ctx.runQuery(target.list, { tenantId, includeDeleted: true, paginationOpts: { numItems: batchSize, cursor: pageCursor } })` returning the `paginator` page `{ page, isDone, continueCursor }` of DTOs with their stream versions, one call per batch; the query caps `numItems` at its byte-derived bound, so a batch never reads more than 8 MiB of stream rows however `batchSize` was set (D2, D10, F15, S4, E-24) - step1: `backfillBatch` loads the generation row by ID and its progress row by `by_generation`, and returns without writing when the row is absent, `state` is not `building`, `phase` is not `backfill` or `fence` differs from its argument (E-40, D9) -- step2: it takes its tenant: the cursor's `tenantId`, or `nextTenant(ctx, null)` when the cursor is `null`; when the tenant list is empty the pass is done and the batch goes to step 6 (E-49, Law 11) +- step2: it takes its tenant: the cursor's `tenantId`, or `nextTenant(ctx, null)` when the cursor is `null`; when the tenant list is empty the batch has no tenant, reads the gate as step 3 states, and goes to step 6 with the pass done (E-49, Law 11) - step3: it calls the write pause's `gateAllows(ctx, ["all", "tenant:" + tenantId], generationId)`, with `["all"]` alone when it has no tenant, and when the answer is not allowed it parks: it patches the progress row's `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt`, schedules nothing and returns, so the generation waits for `resumeGeneration` after the gate reopens; no batch calls `assertWritable`, because a batch parks its generation rather than failing its scheduled mutation with a thrown refusal (E-8, E-42, D9) - step4: it calls `list` once as `fnEnumerateCall` states, with the cursor's `pageCursor` (D10, E-40) - step5: for each DTO of the page it calls `applyProjection(ctx, target.readModel, { tenantId, dto, versions: [dto.version], mode: "backfill", generations: [thisGeneration] })` for this generation alone, adding one to `rowsWritten` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` for `skipped-newer` or `unchanged` (D9, E-9) -- step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"`, patches the progress row to `phase: "verify"`, `cursor: null`, `misses: 0` and the counts, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) -- verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `phase` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and the progress row to `phase: "idle"` and `cursor: null`, and schedules nothing (D9, E-8, Sc L2-5) -- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `phase` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(limitPurgeBatch)`, deletes each row and adds their number to `rowsPurged`; when it deleted `limitPurgeBatch` rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`; a pass that is not done checkpoints and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"` and the progress row to `phase: "idle"` and `cursor: null` (D19, E-40, E-49, Law 11) +- step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"`, patches the progress row to `phase: "verify"`, `cursor: null`, `misses: 0`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) +- verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `phase` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and the progress row to `phase: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) +- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `phase` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(limitPurgeBatch)`, deletes each row and adds their number to `rowsPurged`; when it deleted `limitPurgeBatch` rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`; a pass that is not done checkpoints and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and the progress row to `phase: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) - purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) - fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `phase` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) - fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string, batchSize?: number): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `phase` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, setting the progress row's `batchSize` to `batchSizeFor(readModel, batchSize)` when one is given, clearing `lastError`, and scheduling the batch of the phase, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) @@ -134,13 +134,13 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its phase is not `idle`, and the operator answers with `resumeGeneration` and a smaller batch, down to 1 (E-40, F1, F9, Probe 6) - occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch; a batch reads the generation row and never writes it outside the end of a pass, and it writes the progress row, which no command reads, so a batch's checkpoint re-runs no command (F1, D9, Probe 6, E-40) - limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) -- limitPurgeBatch: `min(500, floor(4194304 / rowBudgetBytes))` rows deleted per purge batch, 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most 4 MiB (E-40, F13, D19) +- limitPurgeBatch: `min(500, floor(4194304 / rowBudgetBytes))` rows deleted per purge batch, 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) - limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch returns without writing (E-40, F13) - limitBuildingGenerations: at most one generation in `building`, `verifying` or `verified` per read model at a time, so a command writes at most two generations, the active one and that one (E-8, E-9) - limitRollbackPeriod: `[extension]` 7 days by default, stored as `retireAfter` on the retired row; the retired generation is kept and not written, and a rollback at or after `retireAfter` is refused (E-8, D9) - limitGenerationsListed: 20 generations per `getGenerations` answer, the newest first (E-8, F13) - fnHistoryPageCall: `ctx.runQuery(components.orders.queries.order.history, { tenantId, streamId, paginationOpts: { numItems: limitHistoryBatch, cursor: eventCursor } })` the one read of context data a history batch makes beside the `list` call of `numItems` 1 that gives it its next stream, returning a `paginator` page of envelopes in version order that the batch folds through `applyHistoryProjection` (D2, D10, E-9, E-24) -- historyStep3: for a history view a batch replaces steps 3 and 4: when the cursor names no stream it calls `list` with `numItems` 1 from the page cursor to take the next stream and sets `eventCursor` to `null`; it then calls `history` for that stream with `limitHistoryBatch` events from `eventCursor`, calls `applyHistoryProjection` in backfill mode for the building generation with the page, and saves `eventCursor`, or clears `streamId` and advances the page cursor when the page reports `isDone` (D9, E-9, E-24, E-40) +- historyStep3: for a history view a batch replaces steps 4 and 5: when the cursor names no stream it calls `list` with `numItems` 1 from the page cursor to take the next stream and sets `eventCursor` to `null`; it then calls `history` for that stream with `limitHistoryBatch` events from `eventCursor`, calls `applyHistoryProjection` in backfill mode for the building generation with the page, and saves `eventCursor`, or clears `streamId` and advances the page cursor when the page reports `isDone` (D9, E-9, E-24, E-40) - historyVerify: a verify batch of a history view enumerates the sources with `list` and, per stream, reads its last event through `history` with `fromVersion` at the stream's version and `numItems` 1, finds that event's row by `keyOf`, and counts a miss when the row is absent or its `sourceVersions` record a lower version for the stream; a miss is repaired by folding the stream's history in place through the same history batches from `eventCursor` null (D9, E-9, Sc L2-6) - rebuildClass: a read model declares its class at declaration time, per entity for the online path, as a `ReadModel` with its projections, and history across streams for the write pause, as a `HistoryProjection`; `pauseRequired` on the generation row records it, and a history view never rebuilds online because a live fold onto a row backfill has not reached cannot be told apart from a complete one (D9, D18, E-9) - limitHistoryBatch: [extension] `min(500, floor(4 MiB / (2 * rowBudgetBytes)))` events per history batch, 128 at the default 16 KiB row budget and 32 at the 64 KiB ceiling, so one batch reads at most that many events of at most `limitEventBytes` of `spec:context.event-envelope`, 20,480 bytes and the 61 bytes of a stored document's system fields, about 2.5 MiB, one stream row through `list`, and at most one history row per event, at most 2 MiB, and writes at most 2 MiB, under the ceilings with the 1 second timeout in mind; the count is stored on the progress row as its `batchSize` and lowered by `resumeGeneration` like the online bound (E-40, E-9, E-24, F13) @@ -162,7 +162,7 @@ And writes {writes:"never stopped"|"reopen only after verification"|"reopen afte ## Verification — reviewed - A reviewer confirms that every batch function states its bound and that the bounds sit below the F13 ceilings with the 1 second timeout in mind. -- A reviewer confirms that no rebuild function calls a context operation, schedules an action or writes anything but read-model rows, the generation row, the progress row and the gate. +- A reviewer confirms that no rebuild function calls a context operation, schedules an action or writes anything but read-model rows, the generation row, the progress row, the gate and the gate's audit records. - A reviewer confirms that the version compare in `applyProjection` is the only thing that prevents a stale overwrite and that L2-5's example asserts it against a concurrent live command. - A reviewer confirms that a batch patches the generation row only in the mutation that ends its pass, and that no command or query over a read model reads the progress row. - A reviewer confirms that every entry that changes a generation calls `assertOperator` first and writes the operator on the row, and that no rebuild function reads a grant or `ctx.auth`. diff --git a/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md b/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md index dfec543..19af9de 100644 --- a/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md +++ b/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md @@ -35,4 +35,4 @@ And writes {writes: "reopen only after verification"} - The test asserts that `PlaceOrder` and `CancelOrder` are refused with `writePaused` because their declarations' `writes` name a closed source, that the error's `data.kind` is `"transient"`, and that nothing is stored. - The test interrupts after the second history batch, asserts the gate is still closed, resumes with `resumeGeneration`, and asserts the generation reaches `verified` with zero misses. - The test asserts that every row of the new generation equals the fold of its order's events and its stock items' allocation events read from the two journals through `history`, and that its `sourceVersions` hold each source stream's current version. -- The test asserts that a `PlaceOrder` sent before `resumeGate` is still refused, then switches, resumes both source scopes, and asserts that the same command now succeeds and that the active generation is the new one. +- The test asserts that a `PlaceOrder` sent once the generation is `verified` and before the switch is still refused, then switches, which resumes both source scopes, and asserts that the same command now succeeds and that the active generation is the new one. diff --git a/design/specs/application/restore.sdp.md b/design/specs/application/restore.sdp.md index 43ea176..abfb693 100644 --- a/design/specs/application/restore.sdp.md +++ b/design/specs/application/restore.sdp.md @@ -93,7 +93,7 @@ The restore itself is a platform operation outside any mutation. Everything the - convexSurface: `startRestoreRun`, `streamCheckBatch`, `readModelCheckBatch`, `receiptCheckBatch`, `pendingWorkBatch`, `finishRestoreRun` as internal mutations and `getRestoreRun` as an internal query, all in the parent (E-42, D19) - tableRestoreRuns: `restoreRuns: defineTable({ backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), state: v.union(v.literal("running"), v.literal("accepted"), v.literal("failed")), checks: v.array(v.object({ name: v.string(), cursor: v.optional(v.string()), examined: v.number(), findings: v.array(v.object({ subject: v.string(), detail: v.string() })), state: v.union(v.literal("pending"), v.literal("running"), v.literal("passed"), v.literal("failed")) })), pendingWork: v.array(pendingWorkItemValidator), operator: v.string(), startedAt: v.number(), finishedAt: v.optional(v.number()) })` where `checks` holds five entries, the four checks and the inventory under the name `pendingWork`, whose cursor and state are kept like a check's while its items go to `pendingWork` (E-42, D19) - validatorPendingWorkItem: `const pendingWorkItemValidator = v.object({ kind: v.union(v.literal("generation"), v.literal("closedScope"), v.literal("import")), subject: v.string(), detail: v.string(), generationId: v.optional(v.id("generations")), fence: v.optional(v.number()), resolution: v.optional(v.union(v.literal("resumed"), v.literal("aborted"), v.literal("kept"), v.literal("recorded"))) })` where `subject` is the read model and generation number, the scope key, or the tenant and import ID, `fence` is the generation's fence at inventory time, and `resolution` is written by `finishRestoreRun` (E-42, D19) -- validatorCheckScope: `const checkScopeValidator = v.union(v.object({ kind: v.literal("tenant"), tenantId: v.string() }), v.object({ kind: v.literal("all") }))` naming which tenants a check covers; under `all` a check batch iterates the application's tenant list the way the rebuild's `tenantIteration` does, one tenant named on every context call, and never calls a context without a tenant (E-42, D11, Law 11) +- validatorCheckScope: `const checkScopeValidator = v.union(v.object({ kind: v.literal("tenant"), tenantId: v.string() }), v.object({ kind: v.literal("all") }))` naming which tenants a check covers; under `all` a check batch iterates the tenant list through `nextTenant`, one tenant at a time, as the rebuild's `step2` and `step6` do, one tenant named on every context call, and never calls a context without a tenant (E-42, D11, Law 11) - indexRestoreRunsByStarted: `.index("by_started", ["startedAt"])` (E-42) - indexRestoreRunsByStartedUse: the operator's listing of runs, newest first (E-42, D19) - fnStartRestoreRun: `export const startRestoreRun = internalMutation({ args: { backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), operator: v.string() }, returns: v.id("restoreRuns"), handler })` an operator entry of `spec:command.actor-and-scope`: it calls `assertOperator` first and records the stated operator on the run record, refusing when `MAINTENANCE_MODE` is not `restore` (E-42, E-48, D19) @@ -102,7 +102,7 @@ The restore itself is a platform operation outside any mutation. Everything the - fnReceiptCheckBatch: `export const receiptCheckBatch = internalMutation({ args: { runId: v.id("restoreRuns"), scope: checkScopeValidator }, returns: v.boolean(), handler })` reading 25 receipts by tenant from the receipt cursor and calling `ctx.runQuery(components.orders.queries.operations.byOperation, { tenantId, operationId })` per receipt for each context the receipt's versions name (E-42, D6, D19) - fnPendingWorkBatch: `export const pendingWorkBatch = internalMutation({ args: { runId: v.id("restoreRuns") }, returns: v.boolean(), handler })` which on its first call, when the inventory's cursor is empty, reads the gate document and appends one `closedScope` item per closed entry, then reads `by_read_model_state` for each read model the parent's declarations name and each of `building`, `verifying` and `verified` and appends one `generation` item per row with its state and fence, and on every call pages the `imports` table with `.paginate` in creation order, `limitPendingWorkBatch` rows at a time, appending one `import` item per row in `running`; it saves the page cursor on its `checks` entry and returns whether the paging is done (E-42, F12, D19, E-41) - fnFinishRestoreRun: `export const finishRestoreRun = internalMutation({ args: { runId: v.id("restoreRuns"), operator: v.string() }, returns: v.null(), handler })` an operator entry of `spec:command.actor-and-scope`, which calls `assertOperator` first and passes the stated operator to `resumeChain`, marking the run failed when any check has a finding; otherwise refusing with a plain error while `MAINTENANCE_MODE` is still `restore` or the inventory has not finished, and then, for every `generation` item whose row is still in `building` or `verifying` at the inventoried fence, calling the rebuild's `resumeChain` in this mutation, which bumps the fence and schedules the next batch from the saved cursor, writing `resumed` on the item, `aborted` on one whose row is `aborted`, `kept` on a `closedScope` item still closed, `recorded` on an `import` item, and marking the run accepted with the finish time (E-42, E-48, D19, F8, F12, Sc L2-8) -- doorExemption: `startRestoreRun`, `streamCheckBatch`, `readModelCheckBatch`, `receiptCheckBatch`, `pendingWorkBatch` and `finishRestoreRun` do not read the gate through `assertWritable` or `gateAllows` and write only `restoreRuns`, except that `finishRestoreRun`'s accepted branch also patches the generation rows it resumes; `startRestoreRun` refuses when `MAINTENANCE_MODE` is not `restore`, and each later mutation refuses when its run is not `running`, so no operational row is written outside a restore and no business table is written by them at all (E-42, D19) +- doorExemption: `startRestoreRun`, `streamCheckBatch`, `readModelCheckBatch`, `receiptCheckBatch`, `pendingWorkBatch` and `finishRestoreRun` do not read the gate through `assertWritable` or `gateAllows` and write only `restoreRuns`, except that `finishRestoreRun`'s accepted branch also patches the generation rows it resumes and their progress rows; `startRestoreRun` refuses when `MAINTENANCE_MODE` is not `restore`, and each later mutation refuses when its run is not `running`, so no operational row is written outside a restore and no business table is written by them at all (E-42, D19) - checkOrder: journal, domain, read model, receipt, then the inventory; a later check does not start while an earlier one has findings, because a broken journal makes the fold check meaningless, and the inventory runs last because it is not a check and its items never fail the run (E-42, D3) - gapRecord: the run record's `backupTakenAt` is the gap start; the procedure writes no synthetic receipt and invents no event for the gap (D19, Existing systems) - limitCheckBatch: [extension] at most `min(25, floor(2 MiB / budgetBytes))` streams per stream-check batch, which is 8 at the default 256 KiB stream budget and 25 for a stream type whose budget is 80 KiB or less, and at most the journal's `maxFoldEvents` of 500 events folded per batch, enforced through `maxEvents` rather than by a stop after the fold that crossed a bound; a batch that folds one stream reads one fold of at most 500 events of at most `limitEventBytes` of `spec:context.event-envelope` per event, 20,480 bytes and the 61 bytes of a stored document's system fields, under 9.8 MiB, with its stored baseline event of at most `budgetBytes + 4,096 + 61` bytes, plus the `list` page of at most 2 MiB of stream rows, plus the same rows read again when each `rebuild` re-reads its row before folding, at most 2 MiB, where each stored row of a single mapping reads 61 bytes of system fields beyond its `budgetBytes`, 488 bytes for the four rows read twice at the cap, and a derived mapping adds 61 bytes for each of its documents, plus the run record of at most the 1 MiB document limit, at most 16,042,313 bytes in all for a single mapping at the 512 KiB cap on `budgetBytes`, under 15.3 MiB against the one budget that nested queries share, as Probe 4 showed, below the 16 MiB read ceiling; each further stream the batch folds reads its own stored baseline event of at most `budgetBytes + 4,096 + 61` bytes beside the events it shares the 500-event bound with, so this total does not bound a batch that folds several streams; a typical batch folds far less, and the 1 second timeout binds first for a batch that folds one full tail, which is why the check is a resumable batch; 25 receipts per receipt batch, each one component call plus one `list` call; 100 rows per read-model batch with one `list` call capped by the context's byte-derived page; the per-stream calls are the cost Probe 3 measures (E-42, E-24, E-25, F13, S3, S9, Probe 3, Probe 4) diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index e37e8d2..b42f344 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -37,7 +37,7 @@ The write pause is the maintenance gate every writer in a scope reads inside its ## Contract -- There is one gate document per deployment, read by the command pipeline at its step 7 for every command through `assertWritable`, which throws, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes while the restore door is closed, as `spec:application.restore` states (D9, E-8, E-25, E-42) +- There is one gate document per deployment, read by the command pipeline at its step 7 for every command through `assertWritable`, which throws, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states (D9, E-8, E-25, E-42) - A scope is `all`, one tenant, or one source stream type of one context; the gate holds the set of closed scopes, each with a reason, the generation it serves if any, and the operator who closed it (E-8, E-48, D9) - A writer in a closed scope refuses before any write with the transient refusal `writePaused`, whose message carries the scope and the reason; nothing is stored, no operation ID is minted, and the same request key succeeds once the scope is open (D9, D4, Law 6) - A use case is in scope when `all` is closed, when its tenant is closed, or when a stream type its declaration's `writes` names is closed (E-8, E-38, D9) @@ -50,8 +50,8 @@ The write pause is the maintenance gate every writer in a scope reads inside its - `startGeneration` closes one entry per source of a paused view and records its own generation on each in the mutation that inserts the generation row, so every entry that names a generation names a row that exists (D9, E-8) - [extension] The operator entries are `closeGate` and `resumeGate` for the scope `all` and for a tenant scope, and `getGate` and `getGateAudit` to read; each follows the operator entry contract of `spec:command.actor-and-scope`, and a source scope is closed and reopened only by the rebuild (E-8, E-48, D19) - Abort marks the generation aborted and resumes its scopes in one mutation (D9, Sc L2-6) -- The restore procedure cannot use the gate document, because the snapshot replaces it; the restore door keeps every writer out through the environment, which survives a restore because backups exclude environment variables; the restore procedure's own mutations, which write only `restoreRuns` and the generation rows the accepted branch resumes, are the one exemption, and that workflow states it (D19, F12, Sc L2-8, E-42) -- [extension] A closed entry the snapshot brings back is as closed as one written live: an entry that names a generation stays closed until the restore's pending-work step resumes that generation or an operator aborts it, which resumes the scope, and a tenant or `all` entry stays closed until an operator resumes it, so the gate never reopens by itself through a restore either (E-8, E-42, F12, D19) +- The restore procedure cannot use the gate document, because the snapshot replaces it; the restore door keeps every writer out through the environment, which survives a restore because backups exclude environment variables; the restore procedure's own mutations, which write only `restoreRuns` and the generation rows the accepted branch resumes with their progress rows, are the one exemption, and that workflow states it (D19, F12, Sc L2-8, E-42) +- [extension] A closed entry the snapshot brings back is as closed as one written live: an entry that names a generation stays closed while the restore's pending-work step resumes that generation, and reopens only when that generation is switched or an operator aborts it, and a tenant or `all` entry stays closed until an operator resumes it, so the gate never reopens by itself through a restore either (E-8, E-42, F12, D19) - [extension] Gate changes are treated as mandatory audit: every close and every resume inserts one record through `writeOperatorAudit` of `spec:operations.baseline-operations` inside the same mutation and fails closed, so a gate change whose audit insert fails changes nothing: the gate document is as it was and no record exists; the doc fails closed only for mandatory business or security audit and does not name gate changes among them (E-8, E-43, D19) - A local reaction refused by the gate is delayed, not counted: the obligation wrapper reschedules the same attempt without incrementing its attempt count, as `spec:obligations.local-reaction-wrapper` rules, so a pause never turns pending work into needs attention (D13, D9, E-51) @@ -73,7 +73,7 @@ The gate is one document so that a command pays one point read. Scopes are strin - fnCloseGate: `export const closeGate = internalMutation({ args: { scopeKey: v.string(), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `closeScope` with no generation: it calls `assertOperator`, throws `"closeGate takes the scope all or a tenant scope, and " + scopeKey + " is neither"` unless `scopeKey` is `all` or `tenant:` followed by a tenant ID of 1 to `limitIdLength` bytes, throws `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and then calls `closeScope`; it does not read the tenant list, so a tenant may be closed before its first grant; this is also how a meaning migration's sweep is run under a pause (E-8, E-25, E-48, D9, D19) - fnResumeGate: `export const resumeGate = internalMutation({ args: { scopeKey: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `resumeScope`: it calls `assertOperator`, makes the scope check of `closeGate` with `resumeGate` in the message, so a source scope is never reopened ahead of its rebuild, and then calls `resumeScope` (E-8, E-48, D9, Sc L2-6) - fnGetGate: `export const getGate = internalQuery({ args: {}, returns: v.object({ restore: v.boolean(), closed: v.array(closedEntryValidator) }), handler })` answering `restore` true when `MAINTENANCE_MODE` is `restore` and `closed` as the document holds it, empty when there is no document (E-8, E-42, D19) -- fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page (E-8, E-43, D19) +- fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page, where `operatorAuditDoc` is the `operatorAudit` table's document validator with `_id` and `_creationTime` (E-8, E-43, D19) - occSerialization: a writer's mutation reads the gate document, or the index range that holds none; a close that commits first invalidates that read, the writer re-runs, reads the closed gate and refuses; a writer that commits first wrote before the close, which is the consistent cut; a batch scheduled before the close but started after it reads the closed gate like any writer, so no closing state, drain interval or confirmation is needed, and the one-second mutation timeout plays no part (F1, D9, E-8) - limitClosedScopes: at most 16 scopes closed at once, so the gate document stays a few kilobytes and the writer check scans an array in memory (E-8, F13) - limitGateReason: a reason is 1 to 256 bytes of UTF-8, so the refusal's message stays under a kilobyte (E-8, F13) diff --git a/design/specs/command/command-pipeline.sdp.md b/design/specs/command/command-pipeline.sdp.md index 070cd6c..95ab44e 100644 --- a/design/specs/command/command-pipeline.sdp.md +++ b/design/specs/command/command-pipeline.sdp.md @@ -86,7 +86,7 @@ The pipeline is one code path with two registered entries per command. The publi - Step 6, admit: for new intent only, the pipeline evaluates the declaration's admission policy; a refusal throws transient `rateLimited` or `capacity`, nothing is stored, and the same intent may be retried (D6, D4, Sc L1-9) - Step 7, gate and mint the operation: the pipeline reads the maintenance gate once through `assertWritable(ctx, scopesOfUseCase(tenantId, decl.writes))` of `spec:application.write-pause`, which refuses with the transient `writePaused` while a scope of this command is closed, the whole deployment, the tenant or a stream type it writes, or the deployment is in restore, and only then mints the whole `OperationRef`, the operation ID, `causedBy` from the call's `causedBy` argument when the internal entry received one and otherwise of kind `command` with the declaration's name, and the caller's optional correlation ID, and passes it to every context call so that events in different streams relate through it (Vocab: operation, D2, D9, D13, E-8, E-26, E-31) - Step 8, execute: the executor captures the outside facts the decision needs, from parent tables or configuration read in this transaction, and calls each context named by the declaration once through `ctx.runMutation` on the component's sanctioned operation, `components..operations.`, with tenant scope, actor, the operation reference, the command's input as lists and the captured facts; each call is an isolated sub-transaction that commits with the parent and rolls back on throw; a rejection thrown inside a context rolls its sub-transaction back and propagates to the boundary's outermost rethrow, and the executor returns the kernel's `CommittedOutcome` together with the `streams` entries every context call returned, never a rejection value (F2, F3, D3, D10, D7, E-23) -- Step 9, update read models: the pipeline first checks that every `streams` entry the executor returned names a context and stream type in the declaration's `writes`, throwing a plain error, a defect and a technical failure, when one does not, so the list the gate of step 7 was derived from is proven on the first native run; then, for every binding, it reads the generations to write through `generationsToWrite`, throws a plain error that names the command and the read model when there is none, and calls `applyProjection` of `spec:application.projection-contract` for each matching entry with the entry's DTO and version and the list unchanged, in the live-created mode when the entry's `created` flag is set and the live-updated mode otherwise, or `applyAggregate` or `applyHistoryProjection` for a binding of those forms, in the same transaction; it counts the rows those calls insert, replace or delete, a row in each generation once, and after each entry is applied throws a plain error when they pass `limitReadModelWritesPerCommand`; the executor writes no read-model table itself (D8, D9, D12, Law 9, E-7, E-8, E-9, E-23, E-45) +- Step 9, update read models: the pipeline first checks that every `streams` entry the executor returned names a context and stream type in the declaration's `writes`, throwing a plain error, a defect and a technical failure, when one does not, so the list the gate of step 7 was derived from is proven on the first native run; then, once for each read model its bindings name, however many bindings name it, it reads the generations to write through `generationsToWrite` and throws a plain error that names the command and the read model when there is none, and, for every binding, calls `applyProjection` of `spec:application.projection-contract` for each matching entry with the entry's DTO and version and the list unchanged, in the live-created mode when the entry's `created` flag is set and the live-updated mode otherwise, or `applyAggregate` or `applyHistoryProjection` for a binding of those forms, in the same transaction; it counts the rows those calls insert, replace or delete, a row in each generation once, and after each entry is applied throws a plain error when they pass `limitReadModelWritesPerCommand`; the executor writes no read-model table itself (D8, D9, D12, Law 9, E-7, E-8, E-9, E-23, E-45) - Step 10, insert the receipt and the audit record: for a receipted call the pipeline inserts the thin receipt, outcome kind, operation ID, affected IDs derived from the stream versions, and the stream versions, in the same transaction as the lookup of step 5, so two concurrent identical requests serialize under optimistic concurrency and only one effect commits; for a declaration that sets `audit` it then calls `writeAudit` of `spec:operations.baseline-operations`, whose failure throws and rolls the command back (D6, D19, F1, Sc L1-3, E-43) - Step 11, return: the mutation emits one diagnostic record through `emitDiagnostic` inside a try and catch that swallows every error, then returns the outcome kind, the result DTO, the operation ID, the affected IDs, the stream versions and `replayed` false (D4, D8, D2, D19, E-43) - At any step, a throw leaves the mutation uncommitted: the state and journal writes of every context sub-transaction, the read-model writes and the receipt are all rolled back, and nothing is stored as a business outcome (Law 2, D7, F2, Sc L1-2) diff --git a/design/specs/operations/baseline-operations.sdp.md b/design/specs/operations/baseline-operations.sdp.md index 277519d..38e2cc5 100644 --- a/design/specs/operations/baseline-operations.sdp.md +++ b/design/specs/operations/baseline-operations.sdp.md @@ -60,7 +60,7 @@ Security, bounded resource use, useful errors and tests start in Layer 1, so non - Restore starts with external dispatch off, reconciles the gap since the backup and rebuilds schedules from obligations, and external idempotency keys survive restore; the transactional profile's procedure is the restore workflow and the durable profile's additions are the obligation module's (D19, F12) - Accepted deferred work stores a logical handler key and version, stable shims stay while vendor jobs refer to old function handles, and an unsupported version goes to needs attention, never an endless retry; this applies from Layer 3 and its detail is the obligation module's (D19, D13) - Security, bounded resource use, useful errors and tests start in Layer 1 (Thesis) -- Every audit record and every diagnostic record names its tenant, and every operator query over them takes the tenant scope (Law 11, D11) +- Every audit record of a command and every diagnostic record names its tenant, and every operator query over them takes the tenant scope; an `operatorAudit` record of a gate change names a tenant only through a tenant scope, and its query takes the scope (Law 11, D11, E-43) - Operator entries that read audit records are internal functions, so only admin access reaches them, as the operator entry contract of `spec:command.actor-and-scope` rules; each one over tenant data takes the tenant, so a client cannot read another tenant's audit (D11, Law 5, E-48) - [extension] A diagnostic record is emitted per command through a sink that never throws, is bounded to 4 KiB, and holds identifiers, outcome kind, error code, versions and timings only (E-43, D19) - [extension] An audit record is one document written by the command pipeline's step 10 through `writeAudit` for every declaration whose `audit` field is set, after an applied command and after a business failure alike, with every field taken from the command as `auditRecordSources` lists, and the failure of that write throws and rolls back the command (E-43, D19, E-38) @@ -71,7 +71,7 @@ Security, bounded resource use, useful errors and tests start in Layer 1, so non Diagnostics and audit are two paths with opposite failure rules. `emitDiagnostic` runs at the end of the command's mutation, after the business writes, inside a try and catch that swallows every error and increments a counter in memory; nothing it does can throw into the mutation. `writeAudit` runs inside the mutation with the business writes, uses `ctx.db.insert` with a validator, and any failure, including a validator refusal or a document too large, throws and rolls the whole command back. Both are parent code; components neither log nor audit for the parent. -Optimistic-concurrency retries cannot be observed from inside a mutation, because a retried mutation re-runs from the start; the metric comes from the platform's function logs and insights. Rebuild and restore progress are read from the generation and restore run rows. The Layer 3 metrics are listed so the list is complete; they are measured once the obligation module is installed. +Optimistic-concurrency retries cannot be observed from inside a mutation, because a retried mutation re-runs from the start; the metric comes from the platform's function logs and insights. Rebuild progress is read from the generation's progress row and restore progress from the restore run row. The Layer 3 metrics are listed so the list is complete; they are measured once the obligation module is installed. - transactionBoundary: audit is written inside the command's top-level mutation and fails closed; diagnostics are emitted from the same mutation after the business writes and never throw (D19, Sc ALL-1) - convexSurface: `getAuditByOperation`, `getAuditBySubject`, `getAuditByRequestKey` as internal queries for operators, each taking `tenantId` first, and `getGateAudit` of `spec:application.write-pause` over `operatorAudit`; the diagnostic sink is `console.log` of one JSON line, read through the platform's logs (E-43, D19) @@ -102,7 +102,7 @@ Optimistic-concurrency retries cannot be observed from inside a mutation, becaus - metricOldestUnresolvedObligation: age of the oldest unresolved obligation, measured once Layer 3 is installed (D19, D13) - metricRetryAndExhaustion: retry and exhaustion counts, measured once Layer 3 is installed (D19, D13) - metricProviderUncertainty: obligations in needs attention for an ambiguous provider outcome, measured once Layer 3 is installed (D19, D14) -- metricRebuildProgress: batches done, rows written and misses per generation, from the generation registry (D19, D9) +- metricRebuildProgress: batches done, rows written and misses per generation, from the generation's `generationProgress` row in the registry (D19, D9, E-40) - metricRestoreProgress: checks passed, examined counts and findings per restore run, from the restore run record (D19, Sc L2-8) - releaseRule: a release keeps a building generation's cursor, a retired generation's rows and an import's progress row readable; a schema change to any of them ships with a migration that runs before the new code reads them (D19) - limitDiagnosticRecord: 4 KiB per diagnostic record; larger records are truncated, never dropped (E-43, D19) @@ -122,5 +122,5 @@ And the failure is {surfaced:"reported as a diagnostic gap without touching the - A reviewer confirms that `emitDiagnostic` is the only path from a command to logs or metrics, that it cannot throw into the mutation, and that no code path decides a business outcome from a diagnostic. - A reviewer confirms that the command pipeline's step 10 calls `writeAudit` for every declaration whose `audit` field is set, inside the mutation, and that no catch surrounds it. -- A reviewer confirms that every batch in the corpus states its bound and that every audit and personal-data index leads with the tenant. +- A reviewer confirms that every batch in the corpus states its bound and that every index of `auditRecords` and of personal data leads with the tenant, and that the `operatorAudit` index leads with the scope. - A reviewer confirms that a retention and deletion policy document exists before any production data and that it names which claims survive deletion. diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 551f56a..68ea5b7 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -32,7 +32,7 @@ The doc says that native tests prove component and nested-mutation behavior, sch - [extension] A composition that runs natively reads its issuer, its application ID and its key set from the deployment's environment variables, and the harness sets the key set to a data URI that holds the fixture issuer's public key (E-13, S13) - [extension] No caller under test carries the admin key, alone or acting as an identity, because a call made with it passes Convex's visibility check (E-13) - [extension] The fixture issuer is the identity source of a native run and its one named difference from production; it is recorded as the identity source and is not an adjusted configuration (E-13, Sc L2-9) -- [extension] The harness uses the admin key to deploy, to generate code, to set and read environment variables, to read stored data and the function log, and to run fixture functions (E-15) +- [extension] The harness uses the admin key to deploy, to generate code, to set and read environment variables, to read stored data and the function log, to run fixture functions, and to call a composition's internal functions that admin access reaches, its grant mutation and its operator entries (E-15, E-48) - [extension] Every Convex CLI call the harness makes names the backend it started on the command line and gets an environment of its own, so that no variable of the caller's shell and no dotenv file selects another deployment (E-15) - [extension] No generated secret, the instance secret or the admin key, reaches an error message, test output or a record (E-15) - [extension] An assertion about authority, identity or visibility is made through an ordinary client (E-15, E-13) From 6bf8fb7ad6c29ca98c07c8cdaacac18016136304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:24:03 +0200 Subject: [PATCH 018/155] build: fold in the read of CancelOrder by the other family Tests: the cancel keeps placedAt on the order and its summary row, at the native and convex-test tiers; a caller without orders.cancel is refused for an order never placed too, which shows the check comes before Orders decides; the order DTO's validator admits cancelled; the order rebuild is compared with a state written out. Each fails with its rule removed. Spec: a decider returns a rejection and its context's adapter throws it; quantities and prices are safe whole numbers; stockLimitExceeded is named; the call order and the quantities' source are shown at the pure tier, not by the native examples; decidedBy D3 and D6. README: the corpus and the application package counts, the E-46 row's maximum. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 8 ++-- ...ventory-example.cancel-placed-order.sdp.md | 4 +- ...ample.cancel-second-context-rejects.sdp.md | 2 +- ...xample.cancel-without-grant-refused.sdp.md | 1 + .../orders-inventory-example.sdp.md | 11 ++++-- tests/native/cancel-order-steps.ts | 19 +++++++++- ...entory-example.cancel-placed-order.test.ts | 6 ++- ...mple.cancel-second-context-rejects.test.ts | 2 +- ...ample.cancel-without-grant-refused.test.ts | 38 +++++++++++++++---- tests/pure/example-domain.test.ts | 28 +++++++++++--- tests/simulator/example.test.ts | 15 +++++++- 11 files changed, 105 insertions(+), 29 deletions(-) diff --git a/design/README.md b/design/README.md index 6611462..c6560a2 100644 --- a/design/README.md +++ b/design/README.md @@ -2,7 +2,7 @@ ## What this is -The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 168 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. +The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 208 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. The thesis, in the doc's words. Ownership follows bounded contexts. Atomicity follows the business operation. Asynchrony follows a concrete need to defer work. The default operation is one Convex mutation that authorizes a command, makes the domain decision, saves current state with its events, updates essential read models and records the outcome. A second transaction exists only to wait, to spread load, or to reach an external system. The design succeeds when adding domain sophistication does not add infrastructure. @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -208 specs · 5 packs · 107 anchors → 320 nodes · 1123 edges (0 errors, 0 warnings) +208 specs · 5 packs · 107 anchors → 320 nodes · 1125 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -85,7 +85,7 @@ Detail follows the build (decision method, rule 4). Full means interfaces, TypeS | all | `platform`, `laws`, `decisions`, `facts` | [`foundation`](specs/foundation.pack.sdp.md) | 75 | | 0 | `kernel` | [`kernel-and-context`](specs/kernel-and-context.pack.sdp.md) | 7 | | 1 | `context`, `command` | [`kernel-and-context`](specs/kernel-and-context.pack.sdp.md) (19 with the kernel), [`command-pipeline`](specs/command-pipeline.pack.sdp.md) | 12 + 22 | -| 2 | `application`, `operations`, `constraints` | [`application`](specs/application.pack.sdp.md) | 31 | +| 2 | `application`, `operations`, `constraints` | [`application`](specs/application.pack.sdp.md) | 43 | | 3 | `obligations`, `effects` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 22 (of the pack's 43) | | 4 | `processes` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 6 | | 5 | `agents` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 7 | @@ -222,7 +222,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | | E-44 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | `BusinessFailurePolicy` (`carry` or `reject`) per context call, applied in the executor body through the pinned helper `carryOrReject` | | E-45 | [`application.read-models`](specs/application/read-models.sdp.md) | At most 8 component reads per parent query and 4 read-model rows written per command, counted as the rows step 9 inserts, replaces or deletes and checked after each `streams` entry is applied, above which step 9 throws a plain error | -| E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a provisional maximum of 100 lines; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` and `CancelOrder` call Orders first; the stock item keeps totals only and `CancelOrder` releases the cancelled order's own lines; `orderNotFound`, `orderAlreadyCancelled` and `insufficientAllocation` as `CancelOrder`'s rejections; the order allocation history view that Sc L2-6 builds | +| E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a maximum of 100 lines, the bound `PlaceOrder`'s declaration carries; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` and `CancelOrder` call Orders first; the stock item keeps totals only and `CancelOrder` releases the cancelled order's own lines; `orderNotFound`, `orderAlreadyCancelled` and `insufficientAllocation` as `CancelOrder`'s rejections; the order allocation history view that Sc L2-6 builds | | E-47 | [`application.first-experiment`](specs/application/first-experiment.sdp.md) | The `Measurement` record, one per measured run, and how each count is captured: commits from the backend's function log, documents and bytes from the `usageStats` of the command's top-level completion record, the calls per context from a pure test of the use case's executor because the log holds no record of a component call inside a mutation, and a count the backend does not expose recorded as a gap; the documents one `PlaceOrder` reads, expected near N + 6 and counted 3N + 5 on the pinned backend, left for the measurement run to explain | | E-50 | [`obligations.obligation-module`](specs/obligations/obligation-module.sdp.md) | The module is a parent-side library plus two parent tables with a static handler registry, not a component | | E-51 | [`obligations.lifecycle-transitions`](specs/obligations/lifecycle-transitions.sdp.md), [`obligations.local-reaction-wrapper`](specs/obligations/local-reaction-wrapper.sdp.md), [`effects.external-effects`](specs/effects/external-effects.sdp.md) | A body's business failure settles `succeeded` with event evidence; a thrown rejection or a declined report settles `cancelled` with `businessRejection` evidence; backoff 1 s times 2 capped at 1 h; 8 attempts by default; a transient `writePaused` or `capacity` refusal defers the same attempt after `retryAfterMs` or 60 s without counting it | diff --git a/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md b/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md index 9df1416..81e4d8a 100644 --- a/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md @@ -29,6 +29,6 @@ And the number of documents the command wrote is {written: 6} - Runs in the native tier on the production composition; every test owns its disposable backend. - The order is `order-1` with two lines on the stock item `sku-1`, of 2 and 1 units; the test asserts that the stock item's state after the cancel equals its state before `PlaceOrder`, `{ onHand: 5, allocated: 0 }`. -- The test asserts the response `{ kind: "applied", replayed: false, result: { orderId: "order-1", released: [{ stockItemId: "sku-1", quantity: 3 }] } }` with versions naming the order at version 2 and the stock item at version 3, Orders first. -- The six documents are the receipt, the order's state and its `OrderCancelled` event, the stock item's state and its one `AllocationReleased` event of quantity 3, and the summary row, whose `sourceVersions` name the order at version 2; the parent's `getOrder` answers the order with status `cancelled`, its lines, total and `placedAt` unchanged. +- The test asserts the response `{ kind: "applied", replayed: false, result: { orderId: "order-1", released: [{ stockItemId: "sku-1", quantity: 3 }] } }` with versions naming the order at version 2 and the stock item at version 3, Orders first; the order of the two calls, Orders and then Inventory, is shown at the pure tier by the same executor run against a ctx that records each `runMutation` by its function reference, which this native test cannot tell apart. +- The six documents are the receipt, the order's state and its `OrderCancelled` event, the stock item's state and its one `AllocationReleased` event of quantity 3, and the summary row, whose `sourceVersions` name the order at version 2; the parent's `getOrder` answers the order with status `cancelled` and the lines, total and `placedAt` it answered before the cancel, and the summary row keeps that `placedAt`. - The test reads the function log and asserts that the command ran as one top-level mutation, and that the order summary list for `cancelled` holds the order and the list for `placed` does not. diff --git a/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md b/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md index e637665..b929917 100644 --- a/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md @@ -31,5 +31,5 @@ And the number of documents the command wrote is {written: 0} - Runs in the native tier on the production composition; every test owns its disposable backend. - No public command releases stock without cancelling its order, so the setup calls the Inventory context's release operation for the order's 3 units directly, with admin access, as a sanctioned operation of that context; the stock item's state stays the fold of its events, and the two contexts now disagree about the order. -- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "insufficientAllocation", commandType: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 } }`, which only a release planned from the lines the Orders call returned can carry. +- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "insufficientAllocation", commandType: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 } }`; that the release's quantities come from the lines the Orders call returned, and that Orders is called first, is shown at the pure tier by the same executor run against a ctx that records each `runMutation` by its function reference. - The test reads the function log and asserts that the request ran one top-level mutation, recorded as failed with no document written, and that the order's stream stays at version 1 with status `placed` and one event. diff --git a/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md b/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md index f781254..5476c1c 100644 --- a/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md @@ -30,3 +30,4 @@ And the number of documents the command wrote is {written: 0} - Runs in the native tier on the production composition; every test owns its disposable backend. - The refused caller is a second user of the fixture issuer holding `orders.place`, `inventory.receive`, `orders.read` and `inventory.read` in the tenant; the test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "forbidden", commandType: "CancelOrder", message: "The caller may not run CancelOrder in this tenant", details: { reason: "no_grant" } }`. +- The refused caller then sends `CancelOrder` for `order-2`, an order never placed, and the test asserts the same `forbidden` data, which shows the refusal comes before Orders decides: an execution before the check would answer `orderNotFound`. diff --git a/design/specs/application/orders-inventory-example.sdp.md b/design/specs/application/orders-inventory-example.sdp.md index c72f3ad..f6262e3 100644 --- a/design/specs/application/orders-inventory-example.sdp.md +++ b/design/specs/application/orders-inventory-example.sdp.md @@ -6,7 +6,9 @@ readiness: defined relations: refines: spec:application.first-experiment decidedBy: + - spec:decisions.d03-events-only-source-of-next-state - spec:decisions.d04-four-outcomes + - spec:decisions.d06-idempotency-client-and-receipts - spec:decisions.d07-rejections-thrown-not-stored - spec:decisions.d10-contexts-meet-in-parent-use-cases constrainedBy: @@ -57,10 +59,11 @@ The second lifecycle command is `CancelOrder`. The stock item keeps two totals a - **StockAllocated** — The Inventory event recorded per stock item when `PlaceOrder` allocates quantity to an order; it carries the order ID and the quantity (First experiment, D2). - **AllocationReleased** — The Inventory event recorded per stock item when `CancelOrder` releases the cancelled order's lines on it, carrying the order ID and the summed quantity, `{ orderId, quantity }`; folding it subtracts the quantity from the quantity allocated and leaves the quantity on hand (E-46, D2). - **short stock rejection** — The rejection `insufficientStock` that the Inventory context throws when a stock item's available quantity is below the quantity an order asks for, with the message "Cannot allocate N when M are available" and the details `{ requested, available }`; `PlaceOrder` lists the code, nothing commits, and an order that waits for stock would be a different command (D4, D7, E-46). -- **unknown order rejection** — The rejection `orderNotFound` that the order decider throws for a cancel of an order with no event, with the message "The order does not exist" and no details; `CancelOrder` lists the code, and nothing commits (D4, D7, E-46). -- **second cancel rejection** — The rejection `orderAlreadyCancelled` that the order decider throws for a cancel of a cancelled order, with the message "The order is already cancelled" and no details, so a second cancel never reaches Inventory and no stock item has to remember an order; `CancelOrder` lists the code, and nothing commits (D4, D7, E-46). -- **release above allocation rejection** — The rejection `insufficientAllocation` that the stock item decider throws for a release larger than the stock item's quantity allocated, with the message "Cannot release N when M are allocated" and the details `{ requested, allocated }`; a cancel of a placed order does not ask for one, because `PlaceOrder` allocated every line in the mutation that recorded the order and one order is cancelled once, and if the two contexts ever disagree `CancelOrder` lists the code and nothing commits (D4, D7, E-46). -- **invalid quantity rejection** — The rejection `invalidQuantity`. The order decider throws it for a place whose line has a quantity below 1 or not whole or a unit price below 0 or not whole, and for lines whose total passes the largest safe whole number; the stock item decider throws it for a receive, allocate or release of a quantity that is not a positive whole number, with the message "A quantity must be a positive whole number, not N" and the details `{ quantity }` (D4, D7, E-46). +- **unknown order rejection** — The rejection `orderNotFound` that the order decider returns for a cancel of an order with no event, and the Orders context's adapter throws, with the message "The order does not exist" and no details; `CancelOrder` lists the code, and nothing commits (D4, D7, E-46). +- **second cancel rejection** — The rejection `orderAlreadyCancelled` that the order decider returns for a cancel of a cancelled order, and the Orders context's adapter throws, with the message "The order is already cancelled" and no details, so a second cancel never reaches Inventory and no stock item has to remember an order; `CancelOrder` lists the code, and nothing commits (D4, D7, E-46). +- **release above allocation rejection** — The rejection `insufficientAllocation` that the stock item decider returns for a release larger than the stock item's quantity allocated, and the Inventory context's adapter throws, with the message "Cannot release N when M are allocated" and the details `{ requested, allocated }`; a cancel of a placed order does not ask for one, because `PlaceOrder` allocated every line in the mutation that recorded the order and one order is cancelled once, and if the two contexts ever disagree `CancelOrder` lists the code and nothing commits (D4, D7, E-46). +- **invalid quantity rejection** — The rejection `invalidQuantity`, which a decider returns and its context's adapter throws. The order decider returns it for a place whose line has a quantity that is not a safe whole number of at least 1 or a unit price that is not a safe whole number of at least 0, with the message "Line I needs a quantity of at least 1 and a unit price of at least 0, each a whole number" and the details `{ line, stockItemId, quantity, unitPrice }`, and for lines whose total passes the largest safe whole number, with the details `{ total }`; the stock item decider returns it for a receive, allocate or release of a quantity that is not a safe whole number of at least 1, with the message "A quantity must be a positive whole number, not N" and the details `{ quantity }` (D4, D7, E-46). +- **stock limit rejection** — The rejection `stockLimitExceeded` that the stock item decider returns, and the Inventory context's adapter throws, for a receive that would take the quantity on hand past the largest safe whole number, with the message "Cannot receive N on top of the M on hand" and the details `{ quantity, onHand }`; `ReceiveStock` lists the code beside `invalidQuantity`, and nothing commits (D4, D7, E-46). - **stock contention** — Two `PlaceOrder` commands for the same stock item sent concurrently; on the native backend both are applied when the stock covers both, and one is applied and one is rejected `insufficientStock` when it covers only one, and the experiment measures the retry cost of the loser (Sc L1-11, First experiment). - **order summary** — The essential summary: a per-entity read model of the parent with the order's status, line count, total and the time it was placed, `{ orderId, status: "placed" | "cancelled", lineCount, total, placedAt }`, written from the order's DTO by every command that writes the order, so `PlaceOrder` inserts the row with `placed` and `CancelOrder` replaces it with `cancelled`, and listed by status (E-46, D8). - **order allocation history** — A history-dependent cross-stream read model of the parent keyed by order ID, folded as a `HistoryProjection` from `OrderPlaced` in Orders and `StockAllocated` and `AllocationReleased` across the stock item streams the order touched, holding per line when it was allocated and released; it is bound once per source stream type, rebuilds only under the write pause, and exists in the experiment for Sc L2-6, not as an essential read (E-9, E-46, D9, Sc L2-6). diff --git a/tests/native/cancel-order-steps.ts b/tests/native/cancel-order-steps.ts index 83f29a0..4f10b4a 100644 --- a/tests/native/cancel-order-steps.ts +++ b/tests/native/cancel-order-steps.ts @@ -30,6 +30,8 @@ export const orderId = "order-1"; export const neverPlaced = "order-2"; // A second user of the fixture issuer, who holds every grant of the composition except orders.cancel. const lackingUser = "user-2"; +// The client of the user who lacks orders.cancel, once a step has granted it the rest. +const lackingClients = new WeakMap(); export const cancelOrderIdentifier = getFunctionName(api.ordering.cancelOrder); type CancelResponse = FunctionReturnType; type PlaceResponse = FunctionReturnType; @@ -39,6 +41,8 @@ export interface CancelWorld { // The stock item's stream row as it was before PlaceOrder allocated. stockBefore?: Record; placed?: PlaceResponse; + // The time the parent's getOrder answered for the placed order, before any cancel. + placedAt?: number; // The cancel a Given step sent under k-1. first?: CancelResponse; target?: string; @@ -80,6 +84,11 @@ export async function orderPlaced(world: CancelWorld, ordered: number) { }); expect(world.placed).toMatchObject({ kind: "applied", replayed: false }); world.ordered = ordered; + const read = await order.client.query(api.orderQueries.getOrder, { + tenantId, + orderId, + }); + world.placedAt = required(read ?? undefined, "the placed order").placedAt; } export async function orderBefore( world: CancelWorld, @@ -119,13 +128,19 @@ export async function orderBefore( ); expect(outcome).toMatchObject({ kind: "applied" }); } -async function lackingClient(order: OrderWorld): Promise { +export async function lackingClient( + order: OrderWorld, +): Promise { + const known = lackingClients.get(order); + if (known !== undefined) return known; for (const permission of permissions) if (permission !== cancelOrderPermission) await grant(order.backend, lackingUser, permission); - return ordinaryClient(order.backend.url, { + const client = ordinaryClient(order.backend.url, { token: await order.backend.issuer.token(lackingUser), }); + lackingClients.set(order, client); + return client; } export async function cancelSent( world: CancelWorld, diff --git a/tests/native/orders-inventory-example.cancel-placed-order.test.ts b/tests/native/orders-inventory-example.cancel-placed-order.test.ts index 98ba523..3572c63 100644 --- a/tests/native/orders-inventory-example.cancel-placed-order.test.ts +++ b/tests/native/orders-inventory-example.cancel-placed-order.test.ts @@ -80,6 +80,8 @@ bindExample( (row) => row["requestKey"] === "k-1", ), ).toMatchObject([{ operationId: response.operationId }]); + // The cancel keeps the time the order was placed, on the order and on its summary row. + const placedAt = required(world.placedAt, "the time the order was placed"); expect(await order.backend.admin.readTable("orderSummaries")).toMatchObject( [ { @@ -87,6 +89,7 @@ bindExample( sourceVersions: [orderVersion], lineCount: 2, total: 1499, + placedAt, }, ], ); @@ -107,6 +110,7 @@ bindExample( { stockItemId, quantity: 1, unitPrice: 999 }, ], total: 1499, + placedAt, version: orderVersion, }); const list = (status: "placed" | "cancelled") => @@ -116,7 +120,7 @@ bindExample( paginationOpts: { cursor: null, numItems: 10 }, }); expect((await list("cancelled")).page).toMatchObject([ - { orderId, status: "cancelled", placedAt: placed?.placedAt }, + { orderId, status: "cancelled", placedAt }, ]); expect((await list("placed")).page).toEqual([]); }, diff --git a/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts b/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts index e1c649c..30e2d5b 100644 --- a/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts +++ b/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts @@ -31,7 +31,7 @@ bindExample( (): CancelWorld => ({}), steps, async (world) => { - // Only a release planned from the lines the Orders call returned asks for 3. + // The call order and the quantities' source are shown at the pure tier, by the executor itself. expect(rejectionData(world)).toEqual({ kind: "rejection", code: "insufficientAllocation", diff --git a/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts b/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts index 87061bf..0a34cdb 100644 --- a/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts +++ b/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts @@ -4,13 +4,19 @@ import { testAnchorId, } from "@libar-dev/software-delivery-protocol"; import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { ConvexError, type Value } from "convex/values"; import { expect } from "vitest"; import { cancelWithoutGrantRefusedContract as contract } from "../../generated/contracts/application.orders-inventory-example.cancel-without-grant-refused.contract.js"; +import { api } from "../../example/convex/_generated/api.js"; +import { required } from "../../harness/native.js"; import { + lackingClient, + neverPlaced, rejectionData, steps, type CancelWorld, } from "./cancel-order-steps.js"; +import { tenantId } from "./order-use-case.js"; const anchor = specTest({ id: testAnchorId( "test:application.orders-inventory-example.cancel-without-grant-refused", @@ -21,17 +27,33 @@ const anchor = specTest({ }); void anchor; // A second user who holds every other grant of the composition, but not orders.cancel, is refused. +const forbidden = { + kind: "rejection", + code: "forbidden", + commandType: "CancelOrder", + message: "The caller may not run CancelOrder in this tenant", + details: { reason: "no_grant" }, +}; bindExample( contract, (): CancelWorld => ({}), steps, - (world) => { - expect(rejectionData(world)).toEqual({ - kind: "rejection", - code: "forbidden", - commandType: "CancelOrder", - message: "The caller may not run CancelOrder in this tenant", - details: { reason: "no_grant" }, - }); + async (world) => { + expect(rejectionData(world)).toEqual(forbidden); + // The same caller's cancel of an order never placed is refused the same way: had Orders decided + // before the check, it would have answered orderNotFound. + const client = await lackingClient(required(world.order, "the backend")); + const error = await client + .mutation(api.ordering.cancelOrder, { + tenantId, + requestKey: "k-2", + input: { orderId: neverPlaced }, + }) + .then( + () => undefined, + (thrown: unknown) => thrown, + ); + expect(error).toBeInstanceOf(ConvexError); + expect((error as ConvexError).data).toEqual(forbidden); }, ); diff --git a/tests/pure/example-domain.test.ts b/tests/pure/example-domain.test.ts index 0d29597..3906025 100644 --- a/tests/pure/example-domain.test.ts +++ b/tests/pure/example-domain.test.ts @@ -3,6 +3,7 @@ import { codeAnchorId, ref, } from "@libar-dev/software-delivery-protocol"; +import { validate } from "convex-helpers/validators"; import { expect, test } from "vitest"; import { checkInvariants, @@ -23,7 +24,10 @@ import { type StockItemEvent, type StockItemState, } from "../../example/domain/index.js"; -import { orderStream } from "../../example/convex/orders/streams.js"; +import { + orderDtoValidator, + orderStream, +} from "../../example/convex/orders/streams.js"; import { orderSummary } from "../../example/convex/orderSummary.js"; import type { StreamMeta } from "../../src/context/index.js"; // Binds the production composition under example/ to the Spec of its domain. The composition is @@ -169,8 +173,15 @@ test("pure: an order rebuilt from OrderPlaced and OrderCancelled equals the stat "OrderPlaced", "OrderCancelled", ]); - expect(rebuild(orderDecider, events)).toStrictEqual(state); - expect(state).toStrictEqual(cancelled()); + // Written out, not folded: the status the cancel sets and what the order keeps. + const expected: OrderState = { + status: "cancelled", + lines, + total: 597, + placedAt: context.now, + }; + expect(state).toStrictEqual(expected); + expect(rebuild(orderDecider, events)).toStrictEqual(expected); }); test.each([ @@ -257,9 +268,16 @@ test("pure: a placed order's DTO carries its lines, total, time and stream versi expect(() => orderStream.toDto(orderDecider.initial(), meta("order-2")), ).toThrow("Order order-2 is not placed and has no DTO"); + const cancelledDto = orderStream.toDto(cancelled(), { + ...meta("order-1"), + streamVersion: 2, + }); + // The DTO's validator, which the context's queries and operations return through, admits both. + expect(validate(orderDtoValidator, cancelledDto)).toBe(true); expect( - orderStream.toDto(cancelled(), { ...meta("order-1"), streamVersion: 2 }), - ).toMatchObject({ + validate(orderDtoValidator, orderStream.toDto(placed(), meta("order-1"))), + ).toBe(true); + expect(cancelledDto).toMatchObject({ orderId: "order-1", status: "cancelled", lines, diff --git a/tests/simulator/example.test.ts b/tests/simulator/example.test.ts index 5515d15..f566352 100644 --- a/tests/simulator/example.test.ts +++ b/tests/simulator/example.test.ts @@ -511,6 +511,12 @@ test( const t = productionTest(); const user = await placedOrder(t); expect(await stockItem(t)).toMatchObject({ onHand: 5, allocated: 3 }); + const before = await user.query(api.orderQueries.getOrder, { + tenantId: "t-1", + orderId: "order-1", + }); + const placedAt = before?.placedAt; + expect(placedAt).toEqual(expect.any(Number)); const response = await user.mutation( api.ordering.cancelOrder, cancelCall("k-1"), @@ -554,6 +560,7 @@ test( status: "cancelled", lines: [line("sku-1", 2, 250), line("sku-1", 1, 999)], total: 1499, + placedAt, version: orderVersion, }); expect(await summaries(t)).toMatchObject([ @@ -564,7 +571,7 @@ test( status: "cancelled", lineCount: 2, total: 1499, - placedAt: order?.placedAt, + placedAt, }, ]); const list = (status: "placed" | "cancelled") => @@ -717,6 +724,12 @@ test( message: "The caller may not run CancelOrder in this tenant", details: { reason: "no_grant" }, }); + // An order never placed is refused the same way: the check comes before Orders decides. + expect( + await errorData( + other.mutation(api.ordering.cancelOrder, cancelCall("k-2", "order-2")), + ), + ).toMatchObject({ code: "forbidden", commandType: "CancelOrder" }); expect(await stockItem(t)).toMatchObject({ allocated: 3 }); expect(await summaries(t)).toMatchObject([{ status: "placed" }]); expect( From b181cb0ccdb8b56625bd75ea98e8e0adb931fb8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:28:28 +0200 Subject: [PATCH 019/155] test: bind scheduler observations to example parameters --- ...uled-functions.probe-7-fresh-import.sdp.md | 14 +- ...d-functions.probe-7-in-place-import.sdp.md | 19 +-- ...exclude-pending-scheduled-functions.sdp.md | 14 +- ...tions-have-limits.probe-7-arguments.sdp.md | 23 ++- .../facts/f13-transactions-have-limits.sdp.md | 9 +- ...able-shows-failed-runs.probe-7-scan.sdp.md | 16 +- ...le-shows-failed-runs.probe-7-states.sdp.md | 14 +- ...d-functions-table-shows-failed-runs.sdp.md | 11 +- design/specs/facts/probe-plan.sdp.md | 2 +- design/specs/platform/native-harness.sdp.md | 2 +- harness/admin.ts | 1 + ...led-functions.probe-7-fresh-import.test.ts | 67 ++++++-- ...-functions.probe-7-in-place-import.test.ts | 100 ++++++++++-- ...ions-have-limits.probe-7-arguments.test.ts | 148 ++++++++++++++++-- ...ble-shows-failed-runs.probe-7-scan.test.ts | 77 +++++++-- ...e-shows-failed-runs.probe-7-states.test.ts | 65 ++++++-- tests/native/scheduler-cases.ts | 118 +++++++------- tests/native/scheduler-restore.ts | 74 +++++---- 18 files changed, 572 insertions(+), 202 deletions(-) diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md index a146f21..a4d5b68 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md @@ -16,13 +16,15 @@ Probe 7 · native backend tier · temporary copy of the production composition. - outcome: Replacement into a fresh backend restores context data without schedules. (Probe 7) ```gwt -Given the production composition with temporary scheduler functions -When the native backend exercises fresh-import -Then the observation is {result: "documents preserved and schedules absent"} +Given a production composition has parent and context documents, stored scheduler ids and all five scheduler states +When its exported snapshot is imported with replacement into a fresh backend with its own environment +Then documents including ids and creation times in {scopes: "parent,orders,inventory"} equal the exported documents {equal: true} +And the scheduler row counts are parent {parent: 0}, Orders {orders: 0} and Inventory {inventory: 0} +And the destination environment is unchanged {unchanged: true} +And each stored id validates as an argument and on a later write {valid: true}, system.get is null {missing: true}, and cancel succeeds {canceled: true} ``` ## Verification — executable -- The archive carries parent, Orders and Inventory documents with their IDs and creation times, but no scheduler rows. Destination environment variables remain unchanged. Stored scheduler IDs validate at import, on a later write and as an argument; system.get returns null and cancel succeeds without a row. These are expectations before the first run. Hosted dashboard restore, Workpool and Workflow are not run. -- Bound values are expectations written before the first native run; measured durations and sizes are recorded, not asserted. -- The first native run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` held the original bound values. +- The value is observed, not expected: the expectation written before the first run was restored documents and empty scheduler tables in a fresh destination, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed that result in the parent, Orders and Inventory, with stored scheduler IDs still validating. +- The exported snapshot is the archive. Full document equality includes IDs and creation times. Hosted dashboard restore, file storage, Workpool and Workflow are not run. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md index fd3199c..a9a06df 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md @@ -16,16 +16,17 @@ Probe 7 · native backend tier · temporary copy of the production composition. - outcome: Replacement in place preserves schedules that run against restored data. (Probe 7) ```gwt -Given the production composition with temporary scheduler functions -When the native backend exercises in-place-import -Then the observation is {result: "documents replaced and schedules preserved"} +Given a production composition exports business documents and all five scheduler states before more orders and schedules are created +When replacement imports the exported snapshot in place before the pending reactions are due +Then documents including ids and creation times in {scopes: "parent,orders,inventory"} equal the exported documents {equal: true} +And every scheduler row in each scope is unchanged {unchanged: true} +And later business changes are gone {removed: true} and the destination environment is unchanged {environmentUnchanged: true} +And kept reactions read {value: "exported"} no earlier than their scheduled time {onTime: true} +And each restored scheduler reference resolves to the kept row {found: true} and cancellation leaves state {state: "canceled"} ``` ## Verification — executable -- Every scheduler state in the parent and both contexts survives replacement. A pending reaction created after export still runs at its scheduled time against restored data. A failed reaction created after export remains visible despite having no matching exported data. The system table alone cannot identify which intent belongs to the exported data. Hosted dashboard restore, Workpool and Workflow are not run. -- Bound values are expectations written before the first native run; measured durations and sizes are recorded, not asserted. -- A kept pending row referenced by an imported `v.id("_scheduled_functions")` is canceled after import; the cancellation is expected to prevent that reaction, while the uncanceled row created after export still runs against restored data. -- The first native run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` held the original bound values. -- The uncanceled reaction is expected to commit no earlier than its recorded scheduled time; elapsed waiting time is recorded rather than bounded. -- Another ReceiveStock and PlaceOrder after export change both contexts and the parent receipts and order summaries; replacement must restore the exported documents exactly and remove these later business changes. +- The value is observed, not expected: the expectation written before the first run was replacement of documents with destination schedules kept, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed unchanged scheduler rows in all scopes and kept pending reactions reading exported data at or after their due time. +- The test changes business data in the parent and both contexts before import and fails if it does not reach that boundary. It records execution times; it binds only that execution is no earlier than the due time. +- The exported snapshot is the archive. Hosted dashboard restore, file storage, Workpool and Workflow are not run. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md index a934f88..8645ede 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md @@ -25,12 +25,16 @@ The backup page excludes scheduled functions, code, configuration and environmen - statement: Backups exclude pending scheduled functions (F12) - flavor: convex-fact - target: evidence.status:documented -- measurableBy: S8 https://docs.convex.dev/database/backup-restore and https://docs.convex.dev/database/import-export/import, read 2026-10-02; native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0; `npx vitest run --project native tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts --testTimeout 300000` as part of `evidence/runs/native-20261002T205553Z-592f1dc-97c74164-c3a8-4818-9f77-2f04b1456261.json`, both import cases passed; the same record includes unrelated probe failures (F12, D13, D19) +- measurableBy: S8 https://docs.convex.dev/database/backup-restore and https://docs.convex.dev/database/import-export/import, read 2026-10-02; native examples for replacement into a fresh backend and in place bind the local observations below (F12, D13, D19, Probe 7) ## Design -- nativeRestore: replacement into both destinations restores the observed parent, Orders and Inventory tables exactly; archive entries contain no `_scheduled_functions`; all five scheduler states in each scope remain in place, and none appear in the fresh destination (F12, Probe 7) -- schedulerReferences: a stored `v.id("_scheduled_functions")` imports into the fresh destination, validates as a function argument and on another document write, returns null from `ctx.db.system.get`, and `ctx.scheduler.cancel` succeeds without making a row; in place the same ID resolves to the kept row and cancellation prevents that pending reaction; an uncanceled row still executes against restored data (F12, Probe 7) -- keptReactions: pending mutations created after export survive import in place and execute at their scheduled time against the restored data; a failed row created after export also survives while its document reference does not, so the system table alone does not say which intent belongs to the exported data (F12, D13, D19, Probe 7) +- freshReplacement: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, replacement into a fresh destination restores the parent, Orders and Inventory documents with their IDs and creation times and leaves all three scheduler tables empty (F12, Probe 7) +- inPlaceReplacement: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, replacement in place restores the exported documents and keeps every destination scheduler row in each scope, including the rows created after export (F12, Probe 7) +- exportedArchive: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, archive entries contain the parent and component tables and no `_scheduled_functions` table (F12, Probe 7) +- destinationEnvironment: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, replacement leaves the destination environment variables unchanged in both cases (F12, Probe 7) +- schedulerReferences: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, a stored `v.id("_scheduled_functions")` imports into the fresh destination, validates as a function argument and on another document write, returns null from `ctx.db.system.get`, and `ctx.scheduler.cancel` succeeds without making a row (F12, Probe 7) +- keptReference: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, an imported scheduler ID in place resolves to its kept row and cancellation changes that row to canceled and prevents its pending reaction (F12, Probe 7) +- keptReactions: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, pending mutations created after export survive import in place and execute no earlier than their scheduled time against the restored value `exported` (F12, D13, D19, Probe 7) +- reactionIntent: a failed row created after export survives import in place while the exported documents replace later business changes; the system table alone does not identify which intent belongs to the exported data (F12, D13, D19, Probe 7) - statusScope: local CLI export and replacement are observed at the native backend tier; documented status remains because hosted dashboard backup and restore are not run, and the local observation is not a confirmed defect in that path (F12, Probe 7) -- cleanNativeEvidence: native backend `precompiled-2026-09-28-5c7cb5b`, SHA-256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae`, `npx vitest run --project native tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts tests/native/snapshot-replacement.test.ts`, `evidence/runs/native-20261002T210938Z-8e603b7-a3ef2d06-9562-4bb0-9963-c0b20231532f.json`, commit `8e603b748d6f95aecccfc15a1f9629b0756ec195`, clean true, all six native tests passed; earlier dirty records above retain exploratory expectation history and are not the evidence for the final behavior claims (Probe 7) diff --git a/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md b/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md index 15db15d..6a73304 100644 --- a/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md @@ -16,14 +16,23 @@ Probe 7 · native backend tier · temporary copy of the production composition. - outcome: Scheduled argument limits count encoded values. (Probe 7) ```gwt -Given the production composition with temporary scheduler functions -When the native backend exercises arguments -Then the observation is {result: "16 MiB summed Convex value bytes per mutation; 4 MiB per call warns"} +Given a temporary mutation builds scheduled payloads internally as one string, an array of strings or eight calls +When it grows ASCII and multibyte payloads across the scheduling limits and reads completion warnings +Then the enforced sum is {limitBytes: 16777216} bytes with object overhead {objectBytes: 14} and array element overhead {elementBytes: 2} +And the refusal text is {refusal: "Error: Too large total size of the arguments of scheduled functions from this mutation (limit: 16777216 bytes)"} at {stage: "runAfter"} and the mutation returns {mutationReturned: true} +And one ASCII string accepts {asciiAccepted: 16777202} bytes and refuses {asciiRefused: 16777203} bytes +And one multibyte string accepts {multibyteAccepted: 8388601} copies of é and refuses {multibyteRefused: 8388602} +And an array of {parts: 1024} strings accepts {arrayAccepted: 16775154} content bytes and refuses {arrayRefused: 16775155} +And {calls: 8} calls accept {callAccepted: 2097138} content bytes each and refuse {callRefused: 2097139} on call {refusedCall: 8} +And single calls with {decimalBytes: 8000000} and {binaryBytes: 8388608} content bytes are accepted +And the shorter warning first appears at {warningFirst: 3355444} accounted bytes after {warningBefore: 3355443} without it, naming limit {warningLimit: 4194304} and text {warningText: "[WARN] Large arguments for a single scheduled function from this mutation"} +And the shorter warning accepts accounted sizes {shortSample: 4194254} and {shortLast: 4194304}, and the future hard error warning first accepts {hardFirst: 4194305} with text {hardText: "[WARN] Large arguments for a single scheduled function from this mutation. This will become a hard error in the future"} naming {namedLimit: 4194304} ``` ## Verification — executable -- The original expectation before the first run followed the limits page. The test constructs values inside the handler, tries ASCII and multibyte strings and arrays of small strings, records refusal text and whether runAfter returned, and fails unless it reaches a refusal and a successful boundary neighbor. Hosted limits are not run. -- Original bound values were written before the first native run; the bound values here follow the observed refusal. Measured durations are recorded, not asserted. -- The first run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` accepted calls above 4 MiB with a warning and refused the sum above 16 MiB at `runAfter`; the original 4 MiB hard-limit expectation did not hold. These values are observed. Exact-byte and encoding comparisons extend that observation. -- At the enforced boundary the test binds 16777216 accounted bytes, accepting one ASCII string of 16777202 bytes, 8388601 copies of é, 1024 strings totaling 16775154 content bytes, and eight calls each carrying 2097138 content bytes; their first larger tested neighbors refuse. The tested object costs 14 accounted bytes beside its string, or 14 plus two per array element beside its strings. Newline and quote characters distinguish this from JSON encoding. +- The value is observed, not expected: the expectation written before the first run was a hard 4 MiB per-call limit, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed a 16777216-byte sum enforced at runAfter while larger single calls warn and return. +- The tested objects account for UTF-8 content plus 14 bytes per call and two bytes per array element. Newline and quote characters distinguish this accounting from JSON escapes. +- The value is observed, not expected: the expectation written before the first run of the warning boundary was no per-call warning at 3355443 accounted bytes, the shorter warning from 3355444 through 4194304 and the future hard error form from 4194305, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed those boundaries with the calls still accepted. +- The warning names a future hard error at 4 MiB, which remains the design ceiling per call. +- The test must reach accepted and refused neighbours. Hosted limits are not run. diff --git a/design/specs/facts/f13-transactions-have-limits.sdp.md b/design/specs/facts/f13-transactions-have-limits.sdp.md index efa2aad..6f48711 100644 --- a/design/specs/facts/f13-transactions-have-limits.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.sdp.md @@ -33,11 +33,12 @@ The documented numbers bound batches. Probe 4 observes transaction budgets; Prob - limitFunctionPayload: 16 MiB per function argument set and 16 MiB per return value; a Node action's arguments are capped at 5 MiB (F13, S9) - limitQueryMutationTimeout: 1 second of execution per query or mutation (F13, S9) - limitActionTimeout: 30 minutes in the Convex runtime and 10 minutes in the Node runtime (F13, S9) -- limitScheduling: the limits page states 1000 scheduled functions per mutation, 4 MiB per scheduled argument set, 16 MiB summed arguments per mutation and 1000000 outstanding functions; the scheduling page states 8 MB total, both read 2026-10-02; the pinned native backend warns above 4 MiB per call, accepts calls above both 8 MB and 8 MiB, and refuses the sum above 16 MiB at `runAfter` (F13, S9, Probe 7) +- limitScheduling: the limits page states 1000 scheduled functions per mutation, 4 MiB per scheduled argument set, 16 MiB summed arguments per mutation and 1000000 outstanding functions; the scheduling page states 8 MB total, both read 2026-10-02; the design keeps 4 MiB per call as its ceiling because this release names that limit as a future hard error (F13, S9, Probe 7) - limitSchema: 32 indexes per table, 16 fields per index, 10,000 tables per deployment (F13, S9) - limitNestedCalls: the limits page states no count of nested `runQuery` or `runMutation` calls per function; the page on writing data states that nested calls share the overall transaction limits, and Probe 4 showed the same across a component boundary for bytes read and bytes written; the pinned backend commits a call stack of nine functions and fails the tenth with "Cross component call depth limit exceeded", which is a backend default and not a documented limit (F13, S9, Probe 4) - transactionMetrics: `ctx.meta.getTransactionMetrics()` is asynchronous and returns `used` and `remaining` per metric, and a nested call accepts `transactionLimits`; Probe 4 read the metrics in a parent around a child's read, nested and in a component, and they grew by what the child read (F13, Probe 4) - limitsAreCeilings: platform limits are ceilings, not batch sizes; every bulk operation, sweeper and batch states a bound tested against the pinned Convex version (D19, F13) -- schedulingObservation: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, `npx vitest run --project native tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts --testTimeout 300000` as part of `evidence/runs/native-20261002T205553Z-592f1dc-97c74164-c3a8-4818-9f77-2f04b1456261.json`; the original 4 MiB hard-limit expectation failed; values are constructed inside the mutation, and the caught `runAfter` refusal returns normally with the number of earlier successful schedules, distinguishing this refusal from commit (F13, Probe 7) -- schedulingBytes: native backend `precompiled-2026-09-28-5c7cb5b`, `npx vitest run --project native tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts` as part of `evidence/runs/native-20261002T205857Z-592f1dc-194ec93d-d96c-4eb7-b639-f7e2c630e648.json`, passed exact-byte boundaries: one string, multibyte text, 1024 strings and eight calls each fit at 16777216 accounted bytes and refuse their first larger tested value; for the tested `{ payload: string }` shape accounting is UTF-8 content bytes plus 14 per call, and for `{ payload: string[] }` it is content bytes plus 14 plus two per element per call; three newline bytes count as three, unlike six JSON escape bytes, so this is Convex value accounting, not JSON text length (F13, Probe 7) -- cleanNativeEvidence: native backend `precompiled-2026-09-28-5c7cb5b`, SHA-256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae`, `npx vitest run --project native tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts tests/native/snapshot-replacement.test.ts`, `evidence/runs/native-20261002T210938Z-8e603b7-a3ef2d06-9562-4bb0-9963-c0b20231532f.json`, commit `8e603b748d6f95aecccfc15a1f9629b0756ec195`, clean true, all six native tests passed; earlier dirty records above retain exploratory expectation history and are not the evidence for the final behavior claims (Probe 7) +- schedulingObservation: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, the expectation of a hard 4 MiB per-call limit did not hold; the mutation constructs its values internally and catches the aggregate 16777216-byte refusal at `runAfter`, then returns normally with the number of accepted calls (F13, Probe 7) +- schedulingBytes: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, one ASCII string, multibyte text, 1024 strings and eight calls fit at 16777216 accounted bytes and refuse their first larger tested value; the tested `{ payload: string }` shape costs UTF-8 content bytes plus 14 per call, and `{ payload: string[] }` adds two per element; newline and quote characters distinguish value accounting from JSON text length (F13, Probe 7) +- acceptedSingleCalls: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, single calls with 8000000 and 8388608 content bytes are accepted (F13, Probe 7) +- schedulingWarnings: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, the shorter per-call warning starts at 3355444 accounted bytes, with none at 3355443, and continues through 4194304; at 4194305 it adds "This will become a hard error in the future"; both forms name a limit of 4194304 bytes and the calls remain accepted (F13, Probe 7) diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md index ca9219f..e229584 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md @@ -16,15 +16,15 @@ Probe 7 · native backend tier · temporary copy of the production composition. - outcome: A failed-function scan reads unrelated schedules. (Probe 7) ```gwt -Given the production composition with temporary scheduler functions -When the native backend exercises scan -Then the observation is {result: "4091 rows scan and 4092 exceed 4096 reads; zero document reads counted"} +Given a successful local reaction and a failed reaction share the scheduler with pending calls to several functions +When a query filters the system table for one function and failed state while unrelated schedules grow past the read boundary +Then the last accepted total is {accepted: 4091} and the first refused total is {refused: 4092} with text {refusal: "Too many reads in a single function execution (limit: 4096)"} +And each successful scan reads {documents: 0} documents and {bytes: 0} bytes, adding {queriesPerRow: 1} database query per row plus {extraQueries: 1} +And the table reaches {total: 32001} rows and scans at {documentCeiling: 32000} and above are refused before any document bound is measured ``` ## Verification — executable -- A plain scheduled mutation records a local reaction. The failed-function filter reads unrelated outstanding rows. Metrics and completion usage record the scan cost. Hosted scan limits and seven-day expiry are not run. -- Original bound values were written before the first native run; the bound values here follow the observed refusal. Measured durations are recorded, not asserted. -- The first run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` counted zero document reads at 1000 rows; the next run refused 32000 rows for too many system operations. The original document-read expectation did not hold. The 4091 and 4092 boundary is an expectation derived from the recorded system-operation count before its first run. -- The boundary run on 2026-10-02 observed 4091 rows returning one failed reaction, with `databaseQueries.used` at 4096, and 4092 rows refused by the 4096-read limit; the same refusal also occurred at 32000, while a system-operation timeout occurred at 32001. The bound values are observed, and a plain scan without metric calls is also checked. -- In `evidence/runs/native-20261002T210145Z-592f1dc-1f503d30-03eb-431b-b48b-e946c2866bd5.json`, the plain scan without metric calls also accepted 4091 rows and refused 4092 for the 4096-read limit. Total rows include three terminal rows; outstanding rows are total rows minus three. +- The value is observed, not expected: the expectation written before the first run was that filtered-out rows consumed the 32000-document allowance, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed zero documents and bytes read, with 4091 rows accepted and 4092 refused at 4096 reads in both plain and instrumented scans. +- The scan adds N + 1 database queries to four already used. Three rows are terminal, so the boundary is 4088 accepted and 4089 refused outstanding rows. +- A local scheduled mutation completes. Populating beyond 32000 rows does not show a document boundary because system reads or system-operation time refuse first. Workpool, Workflow and hosted limits are not run. diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md index 0cfd365..ffb1084 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md @@ -16,13 +16,15 @@ Probe 7 · native backend tier · temporary copy of the production composition. - outcome: The five scheduler states remain readable. (Probe 7) ```gwt -Given the production composition with temporary scheduler functions -When the native backend exercises states -Then the observation is {result: "all five states readable"} +Given the parent and Orders and Inventory have temporary functions for pending, held, successful, failed and canceled schedules +When admin access reads each scheduler table before and after a short interval +Then each scope {scopes: "parent,orders,inventory"} contains state kinds {kinds: "canceled,failed,inProgress,pending,success"} +And rows have names, arguments and scheduled times, and completedTime is present exactly for {terminal: "canceled,failed,success"} +And each failed row has error text containing {failure: "Uncaught Error: reaction refused: exported-failed"} +And all scheduler rows remain unchanged across the interval {retained: true} ``` ## Verification — executable -- Native observations include names, arguments, times and failure errors in the parent and both contexts. Completion rows remain readable during the measured interval. Seven-day expiry and hosted retention are not run. -- Bound values are expectations written before the first native run; measured durations and sizes are recorded, not asserted. -- The first native run on 2026-10-02 on `precompiled-2026-09-28-5c7cb5b` held the original bound values. +- The value is observed, not expected: the expectation written before the first run was all five states with completion times only on terminal rows, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed those states and fields in all three scopes. +- The held action supplies inProgress. The test records the retention interval; seven-day retention and expiry and hosted retention are not run. diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md index 2f12191..788b2a3 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md @@ -25,12 +25,11 @@ The scheduled-functions page documents five states and a seven-day completion-re - statement: `_scheduled_functions` shows failed runs for some retention window (F16) - flavor: convex-fact - target: evidence.status:rechecked -- measurableBy: S6 https://docs.convex.dev/scheduling/scheduled-functions, read 2026-10-02; native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0; `npx vitest run --project native tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts`, `evidence/runs/native-20261002T205513Z-592f1dc-c8bbed5d-4726-482f-a875-5cde26210902.json`: states passed and scan expectation failed; doc status Assumed in the doc, states and seven-day retention documented on S6 (F16, D13) +- measurableBy: S6 https://docs.convex.dev/scheduling/scheduled-functions, read 2026-10-02; native examples bind the five states and failed-function scan cost; the seven-day retention window remains documented on S6 (F16, D13, Probe 7) ## Design -- nativeStates: all five lowercase states are observed in each scope; a held action supplies `inProgress`, which is not a mutation state; completed rows remain readable across the recorded short observation interval (F16, Probe 7) -- scanFinding: the first native scan expected filtered-out scheduler rows to consume the 32000-document allowance; at 1000 rows it instead counted zero document reads and zero bytes, while `databaseQueries.used` grew by 1001; a subsequent run at 32000 rows failed with "Your request timed out performing too many system operations." in `evidence/runs/native-20261002T205553Z-592f1dc-97c74164-c3a8-4818-9f77-2f04b1456261.json`; this is an observed system-operation refusal, not evidence that 32000 is the scan boundary (F16, F13, Probe 7) -- scanBoundary: native backend `precompiled-2026-09-28-5c7cb5b`, `npx vitest run --project native tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts` as part of `evidence/runs/native-20261002T205857Z-592f1dc-194ec93d-d96c-4eb7-b639-f7e2c630e648.json`, observed 4091 rows scanning successfully and 4092 refused with "Too many reads in a single function execution (limit: 4096)"; bytes and documents read remain zero, but the scan adds N + 1 database queries to four already used; 32000 and 32001 are populated and refused, so the document ceiling is not the first boundary; the run failed because the expectation named only the system-operation timeout, and the example is rebound to the observed read refusal (F16, F13, Probe 7) -- scanConfirmation: `npx vitest run --project native tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts`, native backend `precompiled-2026-09-28-5c7cb5b`, `evidence/runs/native-20261002T210145Z-592f1dc-1f503d30-03eb-431b-b48b-e946c2866bd5.json`, passed both the instrumented and plain scan at 4091 total rows and refused both at 4092; three rows are terminal, so these correspond to 4088 and 4089 outstanding rows; the admin count also confirms 32001 total rows (F16, F13, Probe 7) -- cleanNativeEvidence: native backend `precompiled-2026-09-28-5c7cb5b`, SHA-256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae`, `npx vitest run --project native tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts tests/native/snapshot-replacement.test.ts`, `evidence/runs/native-20261002T210938Z-8e603b7-a3ef2d06-9562-4bb0-9963-c0b20231532f.json`, commit `8e603b748d6f95aecccfc15a1f9629b0756ec195`, clean true, all six native tests passed; earlier dirty records above retain exploratory expectation history and are not the evidence for the final behavior claims (Probe 7) +- nativeStates: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, all five states appear in the parent, Orders and Inventory; `completedTime` appears exactly on success, failed and canceled rows, and the failed reaction carries its error text; the held action supplies `inProgress` (F16, Probe 7) +- shortRetention: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, the completed rows remain readable and unchanged across a measured short interval; this does not establish seven-day retention or expiry (F16, Probe 7) +- scanObservation: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, the expectation was that filtered-out scheduler rows consume the 32000-document allowance; the release instead reports zero documents and bytes read and adds N + 1 database queries to four already used; both plain and instrumented scans accept 4091 total rows and refuse 4092 with "Too many reads in a single function execution (limit: 4096)" (F16, F13, Probe 7) +- populatedScan: native backend release `precompiled-2026-09-28-5c7cb5b`, Convex 1.46.0, on 2026-10-02, scans with 32000 and 32001 populated rows refuse on the read limit or time out performing too many system operations; the document ceiling is not the first boundary (F16, F13, Probe 7) diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index 6cde207..313cf07 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -20,7 +20,7 @@ relations: Feature · Detail: full transcription · Traces: Probe 1 to Probe 7, F4, F5, F12, F13, F14, F15, F16, F17, S13, E-16. -Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`; Probe 7 observes the states and a short retention interval, scheduled-argument accounting, failed-function scan cost and local CLI replacement, while seven-day expiry and hosted dashboard restore remain open. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, and each result is recorded on its fact; Probe 7 has native observations for the local scheduler and CLI import, with the remaining scope named below; Probe 6 is separate. +Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`; Probe 7 observes the states and a short retention interval, scheduled-argument accounting, failed-function scan cost and local CLI replacement, while seven-day expiry and hosted dashboard restore remain open. The probes are not the first experiment. The experiment builds Layers 0 to 2 and measures cost; the probes settle single facts and can run before it, in a fixture app that is separate from the example app. diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index df37bc6..268d572 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -66,7 +66,7 @@ The harness is plain test support with no Convex function in it. The entries who - backend: [extension] the disposable backend is the `convex-local-backend` binary of one pinned release, checked against a recorded hash; the harness starts it on local ports with fresh storage, a generated instance secret and the admin key the binary derives from it, sets the environment variables, runs the Convex CLI in the composition's project directory to deploy it, and at the end of the test stops the process and removes the storage (E-15, S13) - identity: [extension] a caller is a `ConvexHttpClient` or a `ConvexClient` whose `setAuth` carries a token the harness signs; `auth.config.ts` declares one `customJwt` provider whose `issuer`, `applicationID` and `jwks` are environment variables; the token carries `iss`, `aud`, `sub`, `iat` and `exp` and no other claim, so `ctx.auth.getUserIdentity()` in a public entry returns the identity those claims yield, and `establishActor`, the grants read and the namespace rules run unchanged as long as they read nothing a provider adds beyond them (E-13, S13, D11) -- adminReads: [extension] the harness reads a table of the app or of a component, and the function log's completion records, with the admin key; a table read returns every document with every Convex value intact or fails, never a sample, and is several queries, not one snapshot; a log read returns every completion record after a point the test marks or fails, and a mark taken before a restart cannot be read after it; a completion record carries at least `identifier`, `componentPath`, `requestId`, `executionTime` in seconds, `cachedResult` and `error` (E-15) +- adminReads: [extension] the harness reads a table of the app or of a component, and the function log's completion records, with the admin key; a table read returns every document with every Convex value intact or fails, never a sample, and is several queries, not one snapshot; a log read returns every completion record after a point the test marks or fails, and a mark taken before a restart cannot be read after it; a completion record carries at least `identifier`, `componentPath`, `requestId`, `executionTime` in seconds, `cachedResult`, `error` and `logLines` as an array of strings (E-15) - typeFixtureIssuer: [extension] `interface FixtureIssuer { issuer: string; applicationID: string; jwks: string; token(subject: string): Promise }` (E-13) - typeBackend: [extension] `interface Backend { url: string; adminKey: string; issuer: FixtureIssuer; admin: AdminAccess; stop(): Promise; kill(): Promise; restart(): Promise; dispose(): Promise; facts(): BackendFacts }`; `stop` and `kill` keep the storage; `dispose` alone removes it and is final, so a disposed backend cannot be started again; `adminKey` is there for the one example that shows why no caller carries it (E-15) - typeAdminAccess: [extension] `interface AdminAccess { deploy(): Promise; deployTemporary(directory: string): Promise<{ command: string[]; stdout: string; stderr: string }>; codegen(): Promise; setEnvironment(variables: Record): Promise; environment(): Promise>; run(name: string, args?: Record, options?: { component?: string }): Promise; readTable(table: string, options?: { component?: string; pageSize?: number }): Promise[]>; writeTable(table: string, write: { insert: Record } | { patch: string; fields: Record } | { delete: string }, options?: { component?: string }): Promise; logMark(): Promise; completionsSince(mark: LogMark, until: (records: readonly CompletionRecord[]) => boolean, timeoutMs?: number): Promise; exportSnapshot(path: string): Promise; replaceSnapshot(path: string): Promise }` holds every use of the admin key and takes no identity; the two snapshot members run the Convex CLI's export and import with replacement against this backend, and a test uses them and never the admin key itself; `deployTemporary` deploys a temporary copy, returns its redacted command and clears the plain composition claim (E-15) diff --git a/harness/admin.ts b/harness/admin.ts index 5ae1ac3..4657e3a 100644 --- a/harness/admin.ts +++ b/harness/admin.ts @@ -52,6 +52,7 @@ export interface CompletionRecord { error: string | null; caller: string; usageStats: Record; + logLines: string[]; } export interface AdminAccess { deploy(): Promise; diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts index eda3057..c270dec 100644 --- a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts @@ -1,4 +1,5 @@ -import { expect } from "vitest"; +import { expect, vi } from "vitest"; +import { isDeepStrictEqual } from "node:util"; import { ref, specTest, @@ -7,7 +8,7 @@ import { import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; import { probe7FreshImportContract as contract } from "../../generated/contracts/facts.f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.contract.js"; import { schedulingBackend } from "./scheduler-composition.js"; -import type { SchedulerWorld } from "./scheduler-cases.js"; +import { type SchedulerWorld } from "./scheduler-cases.js"; import { freshImport } from "./scheduler-restore.js"; const anchor = specTest({ id: testAnchorId( @@ -18,16 +19,60 @@ const anchor = specTest({ ), }); void anchor; -bindExample(contract, (): SchedulerWorld => ({}), { - "the production composition with temporary scheduler functions": async ( +vi.setConfig({ testTimeout: 300000 }); +type World = SchedulerWorld>>; + +bindExample(contract, (): World => ({}), { + "a production composition has parent and context documents, stored scheduler ids and all five scheduler states": + async (world) => { + Object.assign(world, await schedulingBackend()); + }, + "its exported snapshot is imported with replacement into a fresh backend with its own environment": + async (world) => { + world.observation = await freshImport(world.backend!, world.directory!); + }, + "documents including ids and creation times in {scopes} equal the exported documents {equal}": + async (world, { scopes, equal }) => { + const o = world.observation!; + for (const scope of scopes.split(",")) { + const entries = (data: typeof o.restored) => + Object.fromEntries( + Object.entries(data).filter(([key]) => key.startsWith(scope + "/")), + ); + expect(Object.keys(entries(o.restored)).length).toBeGreaterThan(0); + expect( + isDeepStrictEqual(entries(o.restored), entries(o.exported)), + ).toBe(equal); + } + }, + "the scheduler row counts are parent {parent}, Orders {orders} and Inventory {inventory}": + async (world, { parent, orders, inventory }) => { + const s = world.observation!.schedules; + expect(s.parent).toHaveLength(parent); + expect(s.orders).toHaveLength(orders); + expect(s.inventory).toHaveLength(inventory); + }, + "the destination environment is unchanged {unchanged}": async ( world, + { unchanged }, ) => { - Object.assign(world, await schedulingBackend()); - }, - "the native backend exercises fresh-import": async (world) => { - world.result = await freshImport(world.backend!, world.directory!); - }, - "the observation is {result}": (world, { result }) => { - expect(world.result).toBe(result); + expect( + isDeepStrictEqual( + world.observation!.environmentAfter, + world.observation!.environment, + ), + ).toBe(unchanged); }, + "each stored id validates as an argument and on a later write {valid}, system.get is null {missing}, and cancel succeeds {canceled}": + async (world, { valid, missing, canceled }) => { + for (const r of world.observation!.references) { + expect(r.error === null).toBe(valid); + expect(r.copies).toHaveLength(2); + expect(r.copies.every((row) => row.dispatchId === r.id)).toBe(valid); + expect(r.read === null).toBe(missing); + expect(r.error === null).toBe(canceled); + expect(r.schedules).toHaveLength(0); + } + expect(world.observation!.references).toHaveLength(3); + }, }); diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts index bd64caa..85635e7 100644 --- a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts @@ -1,4 +1,5 @@ -import { expect } from "vitest"; +import { expect, vi } from "vitest"; +import { isDeepStrictEqual } from "node:util"; import { ref, specTest, @@ -7,7 +8,7 @@ import { import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; import { probe7InPlaceImportContract as contract } from "../../generated/contracts/facts.f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.contract.js"; import { schedulingBackend } from "./scheduler-composition.js"; -import type { SchedulerWorld } from "./scheduler-cases.js"; +import { type SchedulerWorld } from "./scheduler-cases.js"; import { inPlaceImport } from "./scheduler-restore.js"; const anchor = specTest({ id: testAnchorId( @@ -18,16 +19,93 @@ const anchor = specTest({ ), }); void anchor; -bindExample(contract, (): SchedulerWorld => ({}), { - "the production composition with temporary scheduler functions": async ( +vi.setConfig({ testTimeout: 300000 }); +type World = SchedulerWorld>>; + +bindExample(contract, (): World => ({}), { + "a production composition exports business documents and all five scheduler states before more orders and schedules are created": + async (world) => { + Object.assign(world, await schedulingBackend()); + }, + "replacement imports the exported snapshot in place before the pending reactions are due": + async (world) => { + world.observation = await inPlaceImport(world.backend!, world.directory!); + }, + "documents including ids and creation times in {scopes} equal the exported documents {equal}": + async (world, { scopes, equal }) => { + const o = world.observation!; + for (const scope of scopes.split(",")) { + const entries = (data: typeof o.restored) => + Object.fromEntries( + Object.entries(data).filter(([key]) => key.startsWith(scope + "/")), + ); + expect(Object.keys(entries(o.restored)).length).toBeGreaterThan(0); + expect( + isDeepStrictEqual(entries(o.restored), entries(o.exported)), + ).toBe(equal); + } + }, + "every scheduler row in each scope is unchanged {unchanged}": async ( world, + { unchanged }, ) => { - Object.assign(world, await schedulingBackend()); - }, - "the native backend exercises in-place-import": async (world) => { - world.result = await inPlaceImport(world.backend!, world.directory!); - }, - "the observation is {result}": (world, { result }) => { - expect(world.result).toBe(result); + const o = world.observation!; + expect(Object.keys(o.after).sort()).toEqual([ + "inventory", + "orders", + "parent", + ]); + for (const scope of Object.keys(o.after)) + expect(isDeepStrictEqual(o.after[scope], o.before[scope])).toBe( + unchanged, + ); }, + "later business changes are gone {removed} and the destination environment is unchanged {environmentUnchanged}": + async (world, { removed, environmentUnchanged }) => { + const o = world.observation!; + for (const table of [ + "parent/receipts", + "parent/orderSummaries", + "orders/streams", + "orders/events", + "inventory/streams", + "inventory/events", + ]) { + expect(isDeepStrictEqual(o.changedData[table], o.exported[table])).toBe( + false, + ); + expect(isDeepStrictEqual(o.restored[table], o.exported[table])).toBe( + removed, + ); + } + expect(isDeepStrictEqual(o.environmentAfter, o.environment)).toBe( + environmentUnchanged, + ); + }, + "kept reactions read {value} no earlier than their scheduled time {onTime}": + async (world, { value, onTime }) => { + const o = world.observation!; + expect(o.reactions).toHaveLength(3); + for (const r of o.reactions) { + expect(r.kept!.value).toBe(value); + expect(Number(r.kept!._creationTime) >= o.due).toBe(onTime); + expect( + r.effects.filter((row) => row.label === "exported"), + ).toHaveLength(0); + } + }, + "each restored scheduler reference resolves to the kept row {found} and cancellation leaves state {state}": + async (world, { found, state }) => { + const o = world.observation!; + expect(o.references).toHaveLength(3); + for (const r of o.references) { + expect( + isDeepStrictEqual( + r.found, + o.before[r.component]!.find((row) => row._id === r.id), + ), + ).toBe(found); + expect((r.canceled!.state as { kind: string }).kind).toBe(state); + } + }, }); diff --git a/tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts b/tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts index 366f5b7..e1524a8 100644 --- a/tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts +++ b/tests/native/f13-transactions-have-limits.probe-7-arguments.test.ts @@ -7,23 +7,145 @@ import { import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; import { probe7ArgumentsContract as contract } from "../../generated/contracts/facts.f13-transactions-have-limits.probe-7-arguments.contract.js"; import { schedulingBackend } from "./scheduler-composition.js"; -import { argumentsCase, type SchedulerWorld } from "./scheduler-cases.js"; +import { type SchedulerWorld } from "./scheduler-cases.js"; +import { argumentsCase } from "./scheduler-cases.js"; const anchor = specTest({ id: testAnchorId("test:facts.f13-transactions-have-limits.probe-7-arguments"), verifies: ref("spec:facts.f13-transactions-have-limits.probe-7-arguments"), }); void anchor; vi.setConfig({ testTimeout: 300000 }); -bindExample(contract, (): SchedulerWorld => ({}), { - "the production composition with temporary scheduler functions": async ( - world, - ) => { - Object.assign(world, await schedulingBackend()); - }, - "the native backend exercises arguments": async (world) => { - world.result = await argumentsCase(world.backend!); - }, - "the observation is {result}": (world, { result }) => { - expect(world.result).toBe(result); - }, +type World = SchedulerWorld>>; + +bindExample(contract, (): World => ({}), { + "a temporary mutation builds scheduled payloads internally as one string, an array of strings or eight calls": + async (world) => { + Object.assign(world, await schedulingBackend()); + }, + "it grows ASCII and multibyte payloads across the scheduling limits and reads completion warnings": + async (world) => { + world.observation = await argumentsCase(world.backend!); + }, + "the enforced sum is {limitBytes} bytes with object overhead {objectBytes} and array element overhead {elementBytes}": + async (world, { limitBytes, objectBytes, elementBytes }) => { + const o = world.observation!; + for (const a of o.attempts) { + const perCall = + a.utf8Bytes / a.count + + objectBytes + + (a.parts === 1 ? 0 : elementBytes * a.parts); + const accepted = Math.min(a.count, Math.floor(limitBytes / perCall)); + expect(a.returned, JSON.stringify(a)).toBe(accepted); + expect(a.countedBytes).toBe(accepted * perCall); + } + for (const a of [o.exact, o.multibyte, o.many, o.sum]) + expect(a.countedBytes).toBe(limitBytes); + }, + "the refusal text is {refusal} at {stage} and the mutation returns {mutationReturned}": + async (world, { refusal, stage, mutationReturned }) => { + for (const a of world.observation!.attempts) { + expect(a.mutationReturned).toBe(mutationReturned); + if (a.error !== null) { + expect(a.error).toBe(refusal); + expect(a.stage).toBe(stage); + } else expect(a.returned).toBe(a.count); + } + expect(world.observation!.over.error).not.toBeNull(); + }, + "one ASCII string accepts {asciiAccepted} bytes and refuses {asciiRefused} bytes": + async (world, { asciiAccepted, asciiRefused }) => { + const o = world.observation!; + expect(o.exact.utf8Bytes).toBe(asciiAccepted); + expect(o.over.utf8Bytes).toBe(asciiRefused); + expect(o.exact.error).toBeNull(); + expect(o.over.error).not.toBeNull(); + }, + "one multibyte string accepts {multibyteAccepted} copies of \u00e9 and refuses {multibyteRefused}": + async (world, { multibyteAccepted, multibyteRefused }) => { + const o = world.observation!; + expect(o.multibyte.size).toBe(multibyteAccepted); + expect(o.multibyteOver.size).toBe(multibyteRefused); + expect(o.multibyte.error).toBeNull(); + expect(o.multibyteOver.error).not.toBeNull(); + }, + "an array of {parts} strings accepts {arrayAccepted} content bytes and refuses {arrayRefused}": + async (world, { parts, arrayAccepted, arrayRefused }) => { + const o = world.observation!; + for (const a of [o.many, o.manyOver]) expect(a.parts).toBe(parts); + expect(o.many.utf8Bytes).toBe(arrayAccepted); + expect(o.manyOver.utf8Bytes).toBe(arrayRefused); + expect(o.many.error).toBeNull(); + expect(o.manyOver.error).not.toBeNull(); + }, + "{calls} calls accept {callAccepted} content bytes each and refuse {callRefused} on call {refusedCall}": + async (world, { calls, callAccepted, callRefused, refusedCall }) => { + const o = world.observation!; + for (const a of [o.sum, o.sumOver]) expect(a.count).toBe(calls); + expect(o.sum.size).toBe(callAccepted); + expect(o.sumOver.size).toBe(callRefused); + expect(o.sum.error).toBeNull(); + expect(o.sumOver.error).not.toBeNull(); + expect(o.sumOver.returned! + 1).toBe(refusedCall); + }, + "single calls with {decimalBytes} and {binaryBytes} content bytes are accepted": + async (world, { decimalBytes, binaryBytes }) => { + for (const size of [decimalBytes, binaryBytes]) { + const a = world.observation!.attempts.find( + (a) => + a.size === size && + a.count === 1 && + a.parts === 1 && + a.character === "a", + ); + expect(a).toBeDefined(); + expect(a!.error).toBeNull(); + expect(a!.returned).toBe(1); + } + }, + "the shorter warning first appears at {warningFirst} accounted bytes after {warningBefore} without it, naming limit {warningLimit} and text {warningText}": + async ( + world, + { warningFirst, warningBefore, warningLimit, warningText }, + ) => { + const o = world.observation!; + expect(o.warningFirst.countedBytes).toBe(warningFirst); + expect(o.warningBefore.countedBytes).toBe(warningBefore); + expect( + o.warningBefore.logLines.some((l) => l.startsWith(warningText)), + ).toBe(false); + expect(o.warningFirst.logLines).toContain( + `${warningText} (actual: ${warningFirst} bytes, limit: ${warningLimit} bytes).`, + ); + expect(o.warningFirst.error).toBeNull(); + expect(o.warningFirst.returned).toBe(1); + }, + "the shorter warning accepts accounted sizes {shortSample} and {shortLast}, and the future hard error warning first accepts {hardFirst} with text {hardText} naming {namedLimit}": + async ( + world, + { shortSample, shortLast, hardFirst, hardText, namedLimit }, + ) => { + for (const accounted of [shortSample, shortLast, hardFirst]) { + const a = world.observation!.attempts.find( + (a) => + a.utf8Bytes === accounted - 14 && + a.count === 1 && + a.parts === 1 && + a.character === "a", + )!; + expect(a).toBeDefined(); + expect(a.error).toBeNull(); + expect(a.returned).toBe(1); + expect(a.countedBytes).toBe(accounted); + const text = + accounted === hardFirst + ? hardText + : hardText.replace( + ". This will become a hard error in the future", + "", + ); + expect(a.logLines).toContain( + `${text} (actual: ${accounted} bytes, limit: ${namedLimit} bytes).`, + ); + } + }, }); diff --git a/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts index 7e52830..ca8c2e4 100644 --- a/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts +++ b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts @@ -1,4 +1,4 @@ -import { expect } from "vitest"; +import { expect, vi } from "vitest"; import { ref, specTest, @@ -7,7 +7,8 @@ import { import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; import { probe7ScanContract as contract } from "../../generated/contracts/facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.contract.js"; import { schedulingBackend } from "./scheduler-composition.js"; -import { scanCase, type SchedulerWorld } from "./scheduler-cases.js"; +import { type SchedulerWorld } from "./scheduler-cases.js"; +import { scanCase } from "./scheduler-cases.js"; const anchor = specTest({ id: testAnchorId( "test:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-scan", @@ -17,16 +18,64 @@ const anchor = specTest({ ), }); void anchor; -bindExample(contract, (): SchedulerWorld => ({}), { - "the production composition with temporary scheduler functions": async ( - world, - ) => { - Object.assign(world, await schedulingBackend()); - }, - "the native backend exercises scan": async (world) => { - world.result = await scanCase(world.backend!); - }, - "the observation is {result}": (world, { result }) => { - expect(world.result).toBe(result); - }, +vi.setConfig({ testTimeout: 300000 }); +type World = SchedulerWorld>>; + +bindExample(contract, (): World => ({}), { + "a successful local reaction and a failed reaction share the scheduler with pending calls to several functions": + async (world) => { + Object.assign(world, await schedulingBackend()); + }, + "a query filters the system table for one function and failed state while unrelated schedules grow past the read boundary": + async (world) => { + world.observation = await scanCase(world.backend!); + }, + "the last accepted total is {accepted} and the first refused total is {refused} with text {refusal}": + async (world, { accepted, refused, refusal }) => { + const o = world.observation!; + const success = o.observations.filter((a) => a.error === null); + const failure = o.observations.find((a) => a.error !== null)!; + expect(success.at(-1)!.count).toBe(accepted); + expect(failure.count).toBe(refused); + expect(failure.error).toContain(refusal); + for (const a of o.plain) { + if (a.count === accepted) { + expect(a.error).toBeNull(); + expect(a.rows).toHaveLength(1); + } else { + expect(a.count).toBe(refused); + expect(a.error).toContain(refusal); + } + } + expect(o.plain).toHaveLength(2); + }, + "each successful scan reads {documents} documents and {bytes} bytes, adding {queriesPerRow} database query per row plus {extraQueries}": + async (world, { documents, bytes, queriesPerRow, extraQueries }) => { + const o = world.observation!; + for (const a of [{ count: 5, ...o.first }, ...o.observations].filter( + (a) => a.error === null, + )) { + const r = a.answer!; + expect(r.rows).toHaveLength(1); + expect(r.after.documentsRead.used - r.before.documentsRead.used).toBe( + documents, + ); + expect(r.after.bytesRead.used - r.before.bytesRead.used).toBe(bytes); + expect( + r.after.databaseQueries.used - r.before.databaseQueries.used, + ).toBe(a.count * queriesPerRow + extraQueries); + } + }, + "the table reaches {total} rows and scans at {documentCeiling} and above are refused before any document bound is measured": + async (world, { total, documentCeiling }) => { + const o = world.observation!; + expect(o.actualRows).toBe(total); + for (const count of [documentCeiling, total]) { + const a = o.observations.find((a) => a.count === count)!; + expect(a).toBeDefined(); + expect(a.error).toMatch( + /Too many reads in a single function execution \(limit: 4096\)|too many system operations/, + ); + } + }, }); diff --git a/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts index aa8f7e1..8ca13a6 100644 --- a/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts +++ b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.test.ts @@ -1,4 +1,5 @@ -import { expect } from "vitest"; +import { expect, vi } from "vitest"; +import { isDeepStrictEqual } from "node:util"; import { ref, specTest, @@ -7,7 +8,8 @@ import { import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; import { probe7StatesContract as contract } from "../../generated/contracts/facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-states.contract.js"; import { schedulingBackend } from "./scheduler-composition.js"; -import { statesCase, type SchedulerWorld } from "./scheduler-cases.js"; +import { type SchedulerWorld } from "./scheduler-cases.js"; +import { statesCase } from "./scheduler-cases.js"; const anchor = specTest({ id: testAnchorId( "test:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-states", @@ -17,16 +19,61 @@ const anchor = specTest({ ), }); void anchor; -bindExample(contract, (): SchedulerWorld => ({}), { - "the production composition with temporary scheduler functions": async ( +vi.setConfig({ testTimeout: 300000 }); +type World = SchedulerWorld>>; + +bindExample(contract, (): World => ({}), { + "the parent and Orders and Inventory have temporary functions for pending, held, successful, failed and canceled schedules": + async (world) => { + Object.assign(world, await schedulingBackend()); + }, + "admin access reads each scheduler table before and after a short interval": + async (world) => { + world.observation = await statesCase(world.backend!); + }, + "each scope {scopes} contains state kinds {kinds}": async ( world, + { scopes, kinds }, ) => { - Object.assign(world, await schedulingBackend()); + const o = world.observation!; + expect(Object.keys(o.before).sort()).toEqual(scopes.split(",").sort()); + for (const rows of Object.values(o.before)) + expect( + rows.map((r) => (r.state as { kind: string }).kind).sort(), + ).toEqual(kinds.split(",")); }, - "the native backend exercises states": async (world) => { - world.result = await statesCase(world.backend!); + "rows have names, arguments and scheduled times, and completedTime is present exactly for {terminal}": + async (world, { terminal }) => { + for (const rows of Object.values(world.observation!.before)) + for (const row of rows) { + expect(typeof row.name).toBe("string"); + expect(Array.isArray(row.args)).toBe(true); + expect(typeof row.scheduledTime).toBe("number"); + const kind = (row.state as { kind: string }).kind; + if (Object.hasOwn(row, "completedTime")) + expect(typeof row.completedTime).toBe("number"); + expect(Object.hasOwn(row, "completedTime")).toBe( + terminal.split(",").includes(kind), + ); + } + }, + "each failed row has error text containing {failure}": async ( + world, + { failure }, + ) => { + for (const rows of Object.values(world.observation!.before)) { + const failed = rows.find( + (r) => (r.state as { kind: string }).kind === "failed", + ); + expect((failed!.state as { error: string }).error).toContain(failure); + } }, - "the observation is {result}": (world, { result }) => { - expect(world.result).toBe(result); + "all scheduler rows remain unchanged across the interval {retained}": async ( + world, + { retained }, + ) => { + expect( + isDeepStrictEqual(world.observation!.after, world.observation!.before), + ).toBe(retained); }, }); diff --git a/tests/native/scheduler-cases.ts b/tests/native/scheduler-cases.ts index ea516db..82bede5 100644 --- a/tests/native/scheduler-cases.ts +++ b/tests/native/scheduler-cases.ts @@ -19,28 +19,15 @@ import { } from "./first-experiment-steps.js"; import { ordinaryClient } from "../../harness/clients.js"; import { api } from "../../example/convex/_generated/api.js"; -export interface SchedulerWorld { +export interface SchedulerWorld { backend?: Backend; directory?: string; - result?: string; + observation?: T; } export async function statesCase(backend: Backend) { await initialize(backend, Date.now() + 3600000); const before = await schedulerRows(backend); record("scheduler states", before); - for (const rows of Object.values(before)) { - for (const row of rows) { - expect(typeof row.name).toBe("string"); - expect(Array.isArray(row.args)).toBe(true); - expect(typeof row.scheduledTime).toBe("number"); - const state = row.state as { kind: string; error?: string }; - if (["success", "failed", "canceled"].includes(state.kind)) - expect(typeof row.completedTime).toBe("number"); - else expect(row.completedTime).toBeUndefined(); - if (state.kind === "failed") - expect(state.error).toContain("reaction refused"); - } - } const intervalStart = Date.now(); await sleep(1500); const after = await schedulerRows(backend); @@ -49,8 +36,7 @@ export async function statesCase(backend: Backend) { observedAt: Date.now(), rows: after, }); - expect(after).toEqual(before); - return "all five states readable"; + return { before, after }; } interface Attempt { size: number; @@ -64,6 +50,8 @@ interface Attempt { jsonBytes: number; countedBytes?: number; trim: number; + logLines: string[]; + mutationReturned: boolean; } export async function argumentsCase(backend: Backend) { const attempts: Attempt[] = []; @@ -85,6 +73,8 @@ export async function argumentsCase(backend: Backend) { const observed: Attempt = { ...input, error: null, + logLines: [], + mutationReturned: false, stage: "committed", utf8Bytes: (Buffer.byteLength(value) * parts - @@ -103,6 +93,7 @@ export async function argumentsCase(backend: Backend) { metrics: { scheduledFunctionArgsBytes: { used: number } }; }; Object.assign(observed, { + mutationReturned: true, returned: result.returned, countedBytes: result.metrics.scheduledFunctionArgsBytes.used, error: result.error, @@ -116,6 +107,9 @@ export async function argumentsCase(backend: Backend) { const completions = await backend.admin.completionsSince(mark, (rows) => rows.some((row) => row.identifier === "scheduling:argumentsLimit"), ); + observed.logLines = completions + .filter((row) => row.identifier === "scheduling:argumentsLimit") + .flatMap((row) => row.logLines); record("argument completion records", completions as unknown as Value); attempts.push(observed); measure("argument attempt", { ...observed }); @@ -140,6 +134,26 @@ export async function argumentsCase(backend: Backend) { await attempt(Math.floor((boundary + delta) / 8), 8); for (const character of ["a", "é", "😀", "\n", '"']) for (const parts of [1, 2, 1024]) await attempt(3, 1, character, parts); + await attempt(8000000); + await attempt(8388608); + for (const accounted of [4194254, 4194304, 4194305]) + await attempt(accounted - 14); + // Find the warning onset without assuming that it equals the named limit. + let quiet = 14; + let warned = 4194254; + while (warned - quiet > 1) { + const accounted = Math.floor((quiet + warned) / 2); + const observed = await attempt(accounted - 14); + if ( + observed.logLines.some((line) => + line.includes("Large arguments for a single scheduled function"), + ) + ) + warned = accounted; + else quiet = accounted; + } + const warningBefore = await attempt(quiet - 14); + const warningFirst = await attempt(warned - 14); const exact = await attempt(16 * 1024 * 1024 - 14); const over = await attempt(16 * 1024 * 1024 - 13); const multibyte = await attempt((16 * 1024 * 1024 - 14) / 2, 1, "é"); @@ -152,25 +166,19 @@ export async function argumentsCase(backend: Backend) { "argument boundary summary", attempts.map((a) => ({ ...a })), ); - for (const a of attempts) { - const perCall = - a.utf8Bytes / a.count + 14 + (a.parts === 1 ? 0 : 2 * a.parts); - const accepted = Math.min(a.count, Math.floor(16777216 / perCall)); - expect(a.stage, JSON.stringify(a)).toBe( - accepted === a.count ? "committed" : "runAfter", - ); - expect(a.returned, JSON.stringify(a)).toBe(accepted); - if (accepted < a.count) - expect(a.error).toContain( - "Too large total size of the arguments of scheduled functions from this mutation (limit: 16777216 bytes)", - ); - else expect(a.error).toBeNull(); - } - for (const a of [exact, multibyte, many, sum]) - expect(a.countedBytes).toBe(16777216); - for (const a of [over, multibyteOver, manyOver, sumOver]) - expect(a.error).not.toBeNull(); - return "16 MiB summed Convex value bytes per mutation; 4 MiB per call warns"; + return { + attempts, + exact, + over, + multibyte, + multibyteOver, + many, + manyOver, + sum, + sumOver, + warningBefore, + warningFirst, + }; } export async function scanCase(backend: Backend) { @@ -224,8 +232,14 @@ export async function scanCase(backend: Backend) { return { answer: answer as { rows: Value[]; - before: { documentsRead: { used: number } }; - after: { documentsRead: { used: number } }; + before: Record< + "documentsRead" | "bytesRead" | "databaseQueries", + { used: number } + >; + after: Record< + "documentsRead" | "bytesRead" | "databaseQueries", + { used: number } + >; } | null, error, }; @@ -234,6 +248,7 @@ export async function scanCase(backend: Backend) { expect(first.answer?.rows).toHaveLength(1); let count = 5; const observations = []; + const plain = []; for (const target of [1000, 4090, 4091, 4092, 4096, 32000, 32001]) { while (count < target) { const batch = Math.min(500, target - count); @@ -256,13 +271,7 @@ export async function scanCase(backend: Backend) { rows, error, }); - if (target === 4091) { - expect(error).toBeNull(); - expect(rows).toHaveLength(1); - } else - expect(error).toContain( - "Too many reads in a single function execution (limit: 4096)", - ); + plain.push({ count: target, rows, error }); } } const actualRows = (await backend.admin.readTable("_scheduled_functions")) @@ -272,24 +281,7 @@ export async function scanCase(backend: Backend) { outstanding: actualRows - 3, }); expect(actualRows).toBe(32001); - for (const observed of observations) { - if (observed.count <= 4091) { - expect(observed.error).toBeNull(); - expect(observed.answer?.rows).toHaveLength(1); - expect( - observed.answer!.after.documentsRead.used - - observed.answer!.before.documentsRead.used, - ).toBe(0); - } else if (observed.count <= 4096) - expect(observed.error).toContain( - "Too many reads in a single function execution (limit: 4096)", - ); - else - expect(observed.error).toMatch( - /Too many reads in a single function execution \(limit: 4096\)|too many system operations/, - ); - } - return "4091 rows scan and 4092 exceed 4096 reads; zero document reads counted"; + return { first, observations, plain, actualRows }; } async function productionData(backend: Backend) { diff --git a/tests/native/scheduler-restore.ts b/tests/native/scheduler-restore.ts index a990727..99e85c2 100644 --- a/tests/native/scheduler-restore.ts +++ b/tests/native/scheduler-restore.ts @@ -1,6 +1,5 @@ import { expect } from "vitest"; import type { Backend } from "../../harness/backend.js"; -import { measure } from "../../harness/native.js"; import { exportData, placeAdditionalOrder } from "./scheduler-cases.js"; import { schedulingBackend, @@ -18,13 +17,13 @@ export async function freshImport(backend: Backend, directory: string) { await fresh.admin.replaceSnapshot(exported.path); const restored = await dataRows(fresh); record("fresh restored data", restored); - expect(restored).toEqual(exported.data); + const schedules = await schedulerRows(fresh); record("fresh scheduler tables", schedules); - for (const rows of Object.values(schedules)) expect(rows).toEqual([]); - expect(await fresh.admin.environment()).toEqual(environment); + const environmentAfter = await fresh.admin.environment(); + const references = []; record("fresh environment after import", await fresh.admin.environment()); - measure("fresh environment preserved", true); + for (const component of scopes) { const stored = await fresh.admin.readTable( "schedulerReferences", @@ -49,19 +48,31 @@ export async function freshImport(backend: Backend, directory: string) { error, row: read === undefined ? null : (read as null), }); - expect(error).toBeNull(); - expect(read).toBeNull(); + const copies = await fresh.admin.readTable( "schedulerReferences", options(component), ); - expect(copies).toHaveLength(2); - expect(copies.every((row) => row.dispatchId === id)).toBe(true); - expect( - await fresh.admin.readTable("_scheduled_functions", options(component)), - ).toEqual([]); + references.push({ + component: component ?? "parent", + id, + read, + error, + copies, + schedules: await fresh.admin.readTable( + "_scheduled_functions", + options(component), + ), + }); } - return "documents preserved and schedules absent"; + return { + exported: exported.data, + restored, + schedules, + environment, + environmentAfter, + references, + }; } export async function inPlaceImport(backend: Backend, directory: string) { const due = Date.now() + 45000; @@ -128,16 +139,18 @@ export async function inPlaceImport(backend: Backend, directory: string) { expect(Date.now()).toBeLessThan(due); const after = await schedulerRows(backend); record("in-place scheduler tables after import", after); - expect(after).toEqual(before); + const restored = await dataRows(backend); record("in-place restored data", restored); - expect(restored).toEqual(exported.data); - expect(await backend.admin.environment()).toEqual(environment); + + const environmentAfter = await backend.admin.environment(); + const references = []; + const reactions = []; record( "in-place environment after import", await backend.admin.environment(), ); - measure("in-place environment preserved", true); + for (const component of scopes) { const id = exported.data[`${component ?? "parent"}/schedulerReferences`]![0]! @@ -152,9 +165,6 @@ export async function inPlaceImport(backend: Backend, directory: string) { id, row: found, }); - expect(found).toEqual( - before[component ?? "parent"]!.find((row) => row._id === id), - ); const canceled = ( await backend.admin.readTable("_scheduled_functions", options(component)) ).find((row) => row._id === id); @@ -162,7 +172,7 @@ export async function inPlaceImport(backend: Backend, directory: string) { component: component ?? "parent", row: canceled!, }); - expect(canceled?.state).toEqual({ kind: "canceled" }); + references.push({ component: component ?? "parent", id, found, canceled }); } for (const component of scopes) { await expect @@ -173,12 +183,10 @@ export async function inPlaceImport(backend: Backend, directory: string) { "schedulerEffects", options(component), ) - ).filter((row) => row.label === "after-export"), + ).filter((row) => row.label === "after-export").length, { timeout: 55000, interval: 200 }, ) - .toEqual([ - expect.objectContaining({ label: "after-export", value: "exported" }), - ]); + .toBe(1); const effects = await backend.admin.readTable( "schedulerEffects", options(component), @@ -190,12 +198,22 @@ export async function inPlaceImport(backend: Backend, directory: string) { effects, }); const kept = effects.find((row) => row.label === "after-export"); - expect(Number(kept?._creationTime)).toBeGreaterThanOrEqual(due); - expect(effects.filter((row) => row.label === "exported")).toEqual([]); + reactions.push({ component: component ?? "parent", kept, effects }); } record( "in-place scheduler tables after execution", await schedulerRows(backend), ); - return "documents replaced and schedules preserved"; + return { + exported: exported.data, + changedData, + restored, + before, + after, + environment, + environmentAfter, + references, + reactions, + due, + }; } From 02e9cd704aa19b02b320c5c21b85818bc1af8a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:29:29 +0200 Subject: [PATCH 020/155] design: the main thread's rulings on the check of the generations text The restore's run record keeps who finished it. A restore drives the progress rows of generations in building and verifying only, and resumeGeneration continues a started purge from its progress row. In the backfill mode a null candidate deletes a row only when the row is older, so the version compare has no exception. The Layer 3 repair operations run under the established human or service actor that holds the tenant grant. The progress row's field is pass. The restore's read-model check without a compare mode enters the ledger as open. Co-Authored-By: Claude Opus 5.5 --- design/reviews/consensus-ledger.json | 24 +++++++++++++-- .../application/generation-registry.sdp.md | 8 ++--- .../application/projection-contract.sdp.md | 6 ++-- design/specs/application/rebuild.sdp.md | 30 +++++++++---------- design/specs/application/restore.sdp.md | 4 +-- design/specs/command/actor-and-scope.sdp.md | 2 +- .../obligations/operator-operations.sdp.md | 6 ++-- 7 files changed, 49 insertions(+), 31 deletions(-) diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index badccf3..21ae30c 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -3313,10 +3313,11 @@ ], "issue": "Several Specs require an actor of kind `operator` and the one helper that turns an identity into an actor cannot return one. operator-operations says every operation 'runs under an actor of kind `operator` established by the parent' and its step1 is 'establish the actor from `ctx.auth` in the parent and authorize `obligations:repair` or `obligations:inspect`'. rebuild says its functions 'reach the dashboard through operator functions that authorize an operator actor'. actor-and-scope pins `establishActor` as returning '`null` ..., a `service` actor when the identity's issuer is in `serviceIssuers`, and a `human` actor otherwise'. Grants are read by kind, `.eq(\"principalKind\", actor.kind)`, so a grant with `principalKind: \"operator\"` is never matched by an actor the helper produced, and no Spec says what makes an identity an operator (an issuer, a grant, a deployment admin). The 'operator functions' that front `startGeneration`, `closeGate`, `startRestoreRun` and the other internal mutations taking `actor: actorValidator` are named and nowhere pinned. The same holds for `reviewer`, which is deferred with Layer 4; `operator` is needed from Layer 2.", "fix": "In `spec:command.actor-and-scope`, rule how an operator is established and extend the helper: for example `establishActor(ctx, config: { serviceIssuers: ReadonlySet⟨string⟩; operatorIssuers: ReadonlySet⟨string⟩ })` returning kind `operator` for an operator issuer, or a second helper `establishOperator` that requires a grant of a reserved permission and returns the same identity with kind `operator`. Mark it [extension] under E-37. Then pin one operator entry shape, a public `mutation` or `query` that establishes, authorizes and calls the internal function, and cite it from `spec:obligations.operator-operations` step1, `spec:application.rebuild`, `spec:application.write-pause` and `spec:application.restore`.", - "status": "partially-fixed", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: An operator grant cannot be matched when establishActor returns only human or service, yet the maintenance and obligation contracts require operator kind. The build must show: Pure identity tests and convex-test must produce operator kind only by the ruled trust mechanism and match its grants, leaving ordinary humans and services unable to claim it; source review must align every maintenance caller with that mechanism. S4 on paper, 2026-10-02. `spec:command.actor-and-scope` E-48 answers it for the deployment's operator entries: no function establishes an actor of kind `operator` from a credential, and an entry records the operator admin access states. `spec:obligations.operator-operations` still says its repair operations run under an actor of kind `operator` established by the parent, which `establishActor` cannot return; the main thread ruled that those operations keep their tenant grant, and the actor they run under waits for its ruling.", + "status": "fixed", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: An operator grant cannot be matched when establishActor returns only human or service, yet the maintenance and obligation contracts require operator kind. The build must show: Pure identity tests and convex-test must produce operator kind only by the ruled trust mechanism and match its grants, leaving ordinary humans and services unable to claim it; source review must align every maintenance caller with that mechanism. S4 on paper, 2026-10-02. `spec:command.actor-and-scope` E-48 answers it for the deployment's operator entries: no function establishes an actor of kind `operator` from a credential, and an entry records the operator admin access states. `spec:obligations.operator-operations` still says its repair operations run under an actor of kind `operator` established by the parent, which `establishActor` cannot return; the main thread ruled that those operations keep their tenant grant, and the actor they run under waits for its ruling. Ruled after the check, 2026-10-02. `spec:obligations.operator-operations` now runs the repair operations under the actor the parent establishes, human or service, holding `obligations:repair` or `obligations:inspect` in the tenant, and no Spec establishes an actor of kind `operator`.", "slice": "S4", - "detector": "pure unit test" + "detector": "pure unit test", + "unreviewed": true }, { "round": 3, @@ -4692,6 +4693,23 @@ "note": "Found by the import scout of lane 1d on the native backend, local backend at the pinned release; it enters the ledger through lane 1c, the ledger's one writer in the round. Shown at the native backend tier by `tests/native/snapshot-replacement.test.ts` at commit d53c651 on unit/import-scout; the run and the commands are in report-d-import-scout.md.", "slice": "S5", "detector": "native run of the import scout" + }, + { + "round": 3, + "id": "s4-paper-sdp-restore-read-model-check-compare-mode-undefined", + "lens": "sdp", + "severity": "major", + "files": [ + "specs/application/restore.sdp.md", + "specs/application/projection-contract.sdp.md", + "specs/application/rebuild.sdp.md" + ], + "issue": "`spec:application.restore` `fnReadModelCheckBatch` reuses 'the rebuild's verify logic in compare mode against the active generation', but `spec:application.projection-contract` `typeApplyMode` names no compare mode, and the rebuild's verify pass (`spec:application.rebuild` `verifySteps`) now calls `applyProjection` in backfill mode, which inserts, updates and deletes rows. Read literally, the restore's read-model check would repair the active generation while the restore door is closed and write read-model rows the restore's door exemption says it never writes; read as a comparison, nothing pins the helper, its answer or what counts as a finding.", + "fix": "Pin in `spec:application.restore` a read-only comparison over `projectionOf` and the stored row: for each DTO of the active generation's source, the candidate row of the generation's projection version against the row by `by_key`, a finding for a missing row, an older or differing row, and a row whose DTO projects to `null`, with nothing written but the run record.", + "status": "open", + "note": "S4 on paper, 2026-10-02. Found by the mechanical read of the generations and gate text; older than that text, which made verify a repairing pass. The main thread ruled it is not fixed on paper and goes to the unit that builds the restore. The build must show: at convex-test, a read-model check over an active generation with a missing row, an older row and a row of a deleted subject records three findings and writes no read-model row; a clean generation records none.", + "slice": "S5", + "detector": "judgement" } ], "sessions": [ diff --git a/design/specs/application/generation-registry.sdp.md b/design/specs/application/generation-registry.sdp.md index 3e1d71a..0ebd71d 100644 --- a/design/specs/application/generation-registry.sdp.md +++ b/design/specs/application/generation-registry.sdp.md @@ -42,7 +42,7 @@ The registry is two parent tables. The `generations` table records every generat - Every generation of a read model has one row keyed by read model name and generation number, and generation numbers increase by one per read model, starting at 1 (E-8, D9) - A generation is in exactly one of `building`, `verifying`, `verified`, `active`, `retired`, `aborted` or `purged` (E-8, D9) - At most one generation per read model is `active`, and at most one is in `building`, `verifying` or `verified`; these three states are the in-flight states (E-8, D9) -- The allowed transitions are none to building by `startGeneration`, building to verifying by the backfill batch that ends the enumeration, verifying to verified by the verify batch that ends its pass, verified to active by `switchGeneration`, active to retired by the same `switchGeneration`, retired to verifying by `rollbackGeneration`, building, verifying or verified to aborted by `abortGeneration`, and retired or aborted to purged by the purge batch that ends its pass; no other transition exists, and none leads from nothing to active (E-8, D9) +- The allowed transitions are none to building by `startGeneration`, building to verifying by the backfill batch that ends the enumeration, verifying to verified by the verify batch that ends its pass, verified to active by `switchGeneration`, active to retired by the same `switchGeneration`, retired to verifying by `rollbackGeneration`, building, verifying or verified to aborted by `abortGeneration`, and retired or aborted to purged by the purge batch that ends its pass; no other transition exists, and none leads from nothing to active; `interruptGeneration` and `resumeGeneration` change no state, and `resumeGeneration` continues the pass the progress row names, a started purge of a `retired` or `aborted` generation included (E-8, D9) - An operator entry of `spec:application.rebuild` makes the transitions named after it, a scheduled batch makes the other three, and nothing else writes a generation row (E-8, E-48, D19) - [extension] A read model's first generation is generation 1 in `building`, started by `startGeneration` like any other, and the read model has an active generation only after that generation is verified and switched; on a deployment with no subject the build is one backfill batch and one verify batch for each tenant of the tenant list, and one of each when the list is empty (E-8, D9) - A command that maintains a read model writes the active generation and the in-flight generation if one exists; a retired generation is kept unchanged until it is purged and never written, and a rollback reopens it as `verifying` so that live commands write it again while its verify pass repairs it (D9, E-8) @@ -51,7 +51,7 @@ The registry is two parent tables. The `generations` table records every generat - [extension] The generation row carries the state, the projection version, the fence and who started and last changed it, and nothing a batch writes on its healthy path except the state that the batch ending a pass sets; the progress row carries the cursor, the batch size and the counts, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) - [extension] A generation spans every tenant of the deployment and the registry row names none; the tenant is named on every read-model row and marker, on every context call a batch makes, and in the batch cursor, so no read or write of tenant data runs with an absent tenant as a wildcard, and no generation can be switched for one tenant alone (E-8, D11, Law 11) - An aggregate read model keeps one marker per entity per generation with the contribution last recorded and the source versions it was computed from, and every aggregate write reads and writes the marker in the same mutation as the aggregate row; the marker's `sourceVersions` are compared like a row's, so a DTO older than the marker never adjusts the sum (D9, E-8, E-9) -- [extension] A generation whose progress row is in the phase `backfill`, `verify` or `purge` has at most one scheduled batch driving it, identified by the row's fence; a restore brings the rows back at their checkpoint and drops that batch with every pending scheduled function, so the generation is stranded until `resumeChain` bumps the fence and reschedules from the saved cursor, which the restore procedure's pending-work inventory and accepted branch drive for every such row (E-42, E-8, F12, D19) +- [extension] A generation whose progress row is in the pass `backfill`, `verify` or `purge` has at most one scheduled batch driving it, identified by the row's fence; a restore brings the rows back at their checkpoint and drops that batch with every pending scheduled function, so the generation is stranded until `resumeChain` bumps the fence and reschedules from the saved cursor, which the restore procedure's pending-work inventory and accepted branch drive for every such generation in `building` or `verifying`, and which the operator drives with `resumeGeneration` for a purge the restore stranded, because a purge belongs to a `retired` or `aborted` generation the inventory does not read (E-42, E-8, F12, D19) - The registry is read, never incremented, in the command path; it is not an application-wide counter (D2, E-8) - Registry, progress and marker rows are operational data owned by the parent; they are never written by a context (D2, E-8) - [extension] A command whose declared read model has no generation to write fails as a technical failure, a plain error that names the read model, so a command never commits without its read model (E-8, D8, Law 6) @@ -68,8 +68,8 @@ Tables live in the parent's `schema.ts`, spread from the library's `readModelTab - indexGenerationsByReadModelState: `.index("by_read_model_state", ["readModel", "state"])` (E-8) - indexGenerationsByReadModelStateUse: `generationsToWrite` in every command that maintains the read model, `activeGeneration` in every query over it, and the in-flight check of `startGeneration`, `switchGeneration` and `rollbackGeneration` (D9, E-8) - validatorBatchCursor: `const batchCursorValidator = v.object({ tenantId: v.string(), pageCursor: v.union(v.string(), v.null()), streamId: v.optional(v.string()), eventCursor: v.optional(v.union(v.string(), v.null())) })` the position of a pass, the tenant being iterated and the `paginator` cursor within it, `null` at the start of a tenant; `streamId` and `eventCursor` are set only by the batches of a history view (E-8, E-40, Law 11) -- validatorProgressPhase: `const progressPhaseValidator = v.union(v.literal("backfill"), v.literal("verify"), v.literal("purge"), v.literal("idle"))` naming the pass a batch chain is driving, `idle` when none is (E-40) -- tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), phase: progressPhaseValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position of the pass `phase` names and `null` before that pass has read its first tenant; `batchesDone` counts every batch of the generation that reached its checkpoint, the batch that ends a pass included and a parked batch not, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted (E-40, D9, F13) +- validatorProgressPass: `const progressPassValidator = v.union(v.literal("backfill"), v.literal("verify"), v.literal("purge"), v.literal("idle"))` naming the pass a batch chain is driving, `idle` when none is (E-40) +- tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), pass: progressPassValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position in the pass that `pass` names and `null` before that pass has read its first tenant; `batchesDone` counts every batch of the generation that reached its checkpoint, the batch that ends a pass included and a parked batch not, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted (E-40, D9, F13) - indexProgressByGeneration: `.index("by_generation", ["generationId"])` (E-40) - indexProgressByGenerationUse: the one read a batch and an operator entry make to find a generation's progress row; no command and no query over a read model reads the table (E-40, F1) - tableProjectionMarkers: `projectionMarkers: defineTable({ tenantId: v.string(), readModel: v.string(), generation: v.number(), entityKey: v.string(), aggregateKey: v.string(), contribution: v.number(), sourceVersions: v.array(streamVersionValidator) })` (E-8, D9, Law 11) diff --git a/design/specs/application/projection-contract.sdp.md b/design/specs/application/projection-contract.sdp.md index 6b4e3bb..e3d2b24 100644 --- a/design/specs/application/projection-contract.sdp.md +++ b/design/specs/application/projection-contract.sdp.md @@ -49,8 +49,8 @@ A projection is the named, versioned, deterministic function that turns a contex - In the active generation a live write always lands: a missing row is inserted and an existing row is replaced, in the live-created mode and the live-updated mode alike, because `project` sees the whole DTO and cannot build a partial row, so a subject whose row was deleted gets its row back from the next command that writes it (D9, E-9) - In a generation in `building`, the role `building`, the live-created mode inserts or replaces the row, and the live-updated mode replaces a row the backfill has already written and skips a missing one; in a generation in `verifying` or `verified`, the role `verifying`, the backfill has ended, a missing row is a subject that projected to none, and a live write lands as it does in the active generation (D9, E-8, E-9) - [extension] Every write of a read-model row measures the whole row, the conventions included, with `getConvexSize` and throws a plain error, a technical failure that rolls the command back, when it measures above the read model's `rowBudgetBytes`; a declared budget above the ceiling of `limitRowSize` throws the same way (E-9, F13, Law 6) -- In the backfill mode the one generation the batch names is written whatever its role: the row is inserted if missing, updated if older, and deleted if present when the DTO projects to `null`, which is how a verify pass makes absence right as well as presence (D9, E-8, E-9) -- A projection may return `null` for a DTO that has no row, such as a deleted subject; `applyProjection` then deletes the row if present and does nothing when it is absent, and nothing records the deletion, because a backfill batch reads the source in the same mutation that writes the row, so a batch that read the subject before its deletion committed re-runs under optimistic concurrency and reads the deleted subject, as the rebuild's `occRace` states (E-9, D2, F1) +- In the backfill mode the one generation the batch names is written whatever its role: the row is inserted if missing, updated if older, and, when the DTO projects to `null`, deleted if its `sourceVersions` are older than the input's and left as `skipped-newer` otherwise, which is how a verify pass makes absence right as well as presence (D9, E-8, E-9) +- A projection may return `null` for a DTO that has no row, such as a deleted subject; `applyProjection` then deletes the row if present, in the backfill mode only when the row is older than the input, and does nothing when it is absent, and nothing records the deletion, because a backfill batch reads the source in the same mutation that writes the row, so a batch that read the subject before its deletion committed re-runs under optimistic concurrency and reads the deleted subject, as the rebuild's `occRace` states (E-9, D2, F1) - An aggregate projection contributes a number per entity to an aggregate row and keeps a per-entity marker so live commands and backfill never count one entity twice (D9) - [extension] `applyAggregate` ignores the mode: the marker's `sourceVersions` decide, a marker that is not older than the input makes the write `skipped-newer`, an older or missing marker is written with the new contribution and versions and the difference goes to `value`, and a `null` contribution removes the marker and its recorded contribution; a live update on an entity backfill has not reached is therefore not partial, because `contribute` sees the whole DTO, which is why the missing-row rule of the per-entity path does not apply (E-9, D9, F1) - [extension] A history projection folds events into a row: `keyOf` maps each event to the row it belongs to or to none, and `project` folds the events of one key, in stream-version order within each stream, into the prior row; it is deterministic in the prior row and the events, reads nothing else, and is the one form a read model that depends on history across several streams may take, which is why such a read model rebuilds only under the write pause (E-9, D8, D9, Law 10) @@ -87,7 +87,7 @@ The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape w - tableOrderSummaries: `orderSummaries: defineTable({ ...rowConventions, ...orderSummaryFields }).index("by_key", ["tenantId", "generation", "key"]).index("by_status", ["tenantId", "generation", "status", "placedAt"])` (E-9, E-46, First experiment) - indexByKeyUse: point read by `applyProjection` before every write and by the entity query; `key` is `keyOf(tenantId, dto)`, the application ID of the entity (E-9, D2) - indexByStatusUse: the essential list query of the first experiment, paginated, within the active generation (E-9, First experiment) -- step1: for each generation in `generations`, `applyProjection` takes `projectionOf(readModel, generation.projectionVersion)` and computes `key` and the candidate row with that projection's `keyOf` and `project`; a `null` candidate means delete the row if present, answering `deleted`, and `unchanged` when there is none, in every mode and role (E-9, D8) +- step1: for each generation in `generations`, `applyProjection` takes `projectionOf(readModel, generation.projectionVersion)` and computes `key` and the candidate row with that projection's `keyOf` and `project`; a `null` candidate means delete the row if present, answering `deleted`, and `unchanged` when there is none, in the two live modes and every role; in the backfill mode the version compare of `step3` comes first, so a present row is deleted only when its `sourceVersions` are older than the input's and answers `skipped-newer` otherwise (E-9, D8) - step2: it reads that generation's row by `by_key` (E-9, D9) - step3: it decides by the mode and the generation's role: in the two live modes it inserts a missing row and replaces an existing one, except that live-updated in the role `building` answers `skipped-missing` for a missing row; in the backfill mode it inserts when missing and updates when the existing `sourceVersions` are older, else `skipped-newer`, whatever the role (D9, E-8, E-9) - step4: before a write it throws a plain error when the row measures above `rowBudgetBytes` with `getConvexSize`; then it writes with `ctx.db.insert`, `ctx.db.replace` or `ctx.db.delete` in the same mutation, a written row carrying the generation's `projectionVersion`, which is the version of the projection that built it, and returns the results in generation order (E-9, F1, F13) diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index e8e564c..820f25e 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -83,15 +83,15 @@ The registry, the progress row, the per-entity marker and the gate document are - rule: [extension] A projection version change is a new generation, never an in-place rewrite of the active one (E-8, D8) - rule: [extension] The batch bound is a field of the progress row, a whole number of at least 1 and at most `rebuildBatchCeiling`, and never more than the context's `list` page admits at the stream type's document budget; only `resumeGeneration` changes it (E-40, E-24, E-9, F1, F13) - rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them returns without writing (E-40, E-8, F8) -- An operator calls `startGeneration` for a read model; it inserts the generation row in state `building` with the next generation number, the projection version and fence 1, inserts the progress row in the phase `backfill` with the batch size and a `null` cursor, and schedules the first backfill batch, all in one mutation; a read model with no generation gets generation 1 this way (D9, F8, E-8) +- An operator calls `startGeneration` for a read model; it inserts the generation row in state `building` with the next generation number, the projection version and fence 1, inserts the progress row in the pass `backfill` with the batch size and a `null` cursor, and schedules the first backfill batch, all in one mutation; a read model with no generation gets generation 1 this way (D9, F8, E-8) - From that commit every use case that maintains the read model writes the active generation, when there is one, and the one being built through `applyProjection`, in live-created mode for a stream it created and live-updated mode otherwise (D9, E-9) -- Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building`, the phase is not `backfill` or the fence differs, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) +- Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building`, the pass is not `backfill` or the fence differs, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) - A batch that throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, and a developer error or a timeout ends the chain; the operator's `interruptGeneration` ends a chain on purpose, and `resumeGeneration` schedules a batch from the saved cursor with a new fence and, when asked, a smaller batch; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) - Each verify batch enumerates the sources again from the progress row's cursor, which the end of the backfill reset to the first tenant, and calls `applyProjection` in backfill mode for every DTO, so a missing row is inserted, an older one updated, and the row of a DTO that projects to `null` deleted; each such repair is counted as a miss; when the last tenant's last page is done the generation moves to `verified` and no batch is scheduled (D9, E-8, Sc L2-5) - `switchGeneration` sets the verified generation `active` and, when the read model has an active generation, sets that one `retired` with `retiredAt` and `retireAfter`, in one mutation; every query reads the active generation from the registry, so every subscription on the read model re-runs (D9, F7, F1) - [extension] During the rollback period the retired generation's rows are kept and not written; `rollbackGeneration` reopens the retired generation it is given as `verifying` with a new fence and a `null` cursor and schedules a verify batch, live commands write it again from that commit, its one verify pass repairs every row that is missing, older or left behind by a deleted subject, and the ordinary `switchGeneration` makes it active and retires the generation that served meanwhile (E-8, D9, Sc L2-5) - `abortGeneration` marks a building, verifying or verified generation `aborted`, leaves the active generation untouched, and resumes every closed gate entry that names this generation; an aborted rollback leaves the reopened generation `aborted`, never `retired` again (D9, Sc L2-6, E-8) -- `purgeGeneration` starts the purge of an aborted generation, or of a retired one once `retireAfter` has passed; each purge batch deletes at most `limitPurgeBatch` rows of the generation for one tenant by the `by_key` range, stays on that tenant while a full batch came back, moves to the next tenant of the tenant list otherwise, and marks the generation `purged` after the last tenant; nothing starts a purge by itself (D19, E-40, E-49) +- `purgeGeneration` starts the purge of an aborted generation, or of a retired one once `retireAfter` has passed; each purge batch deletes at most `limitPurgeBatch` rows of the generation for one tenant by the `by_key` range, stays on that tenant while a full batch came back, moves to the next tenant of the tenant list otherwise, and marks the generation `purged` after the last tenant; nothing starts a purge by itself, and a purge whose chain ended, a restore among the causes, is continued by `resumeGeneration` from its progress row (D19, E-40, E-49) - For a history-dependent read model the generation row records `pauseRequired`, and `startGeneration` closes the maintenance gate for every source scope of the read model's bindings in the mutation that inserts the row, recording its own generation on each closed entry, so the gate is closed before the first batch is scheduled and this generation's batches are the one background writer allowed through (D9, E-8, E-38) - While the gate is closed for the sources, every use case whose declaration writes one of them and the journal's baseline driver refuse with `writePaused` before any write, whether or not they maintain the view; a batch scheduled before the close reads the closed gate when it starts and refuses, a batch running when the close commits loses the optimistic-concurrency race on the gate document and re-runs into the refusal, and a batch of a superseded chain refuses because its generation fence is stale; nothing needs to drain (D9, D4, E-8, E-38, F1) - Backfill under the pause enumerates a consistent cut because no source changes: each history batch takes one stream from the enumeration, reads one `history` page of it from the saved event cursor, folds the page into the building generation through `applyHistoryProjection` in backfill mode, and saves the stream and the event cursor on the progress row, moving to the next stream when the page reports `isDone`; an interrupted batch resumes from its cursor with the gate still closed, and the gate never reopens by itself (D9, E-9, Sc L2-6) @@ -107,31 +107,31 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - convexSurface: `startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` as the operator entries, `backfillBatch`, `verifyBatch` and `purgeBatch` as the scheduled batches, all internal mutations, and `getGenerations` as an internal query, in the composition's `rebuild` module; the gate entries are in the write-pause contract (D9, E-8, E-48) - typeRebuildTarget: `type RebuildTarget = { readModel: ReadModel; source: SourceRef; list: FunctionReference<"query", "internal", ListArgs, PaginationResult> }` one per-entity read model the deployment can rebuild, its one source stream type, and the reference to that stream type's `list` query in its context component, such as `components.orders.queries.order.list` (E-8, E-9, D10) - typeRebuildConfig: `type RebuildConfig = { targets: readonly RebuildTarget[]; refs: { backfillBatch: BatchRef; verifyBatch: BatchRef; purgeBatch: BatchRef } }` where `type BatchRef = FunctionReference<"mutation", "internal", { generationId: Id<"generations">; fence: number }, null>` and `refs` are the composition's own registered batches, which a batch schedules by reference (E-40, F8) -- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; then it inserts the generation row with the next number, `state: "building"`, `pauseRequired: false`, since a `RebuildTarget` is a per-entity read model and the paused start of a history view is given only in outline, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `phase: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48) +- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; then it inserts the generation row with the next number, `state: "building"`, `pauseRequired: false`, since a `RebuildTarget` is a per-entity read model and the paused start of a history view is given only in outline, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `pass: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48) - fnBatchSizeFor: `batchSizeFor(readModel: ReadModel, requested?: number): number` throwing `"The batch size must be a whole number of at least 1, and " + requested + " is not"` unless `requested` is absent or an integer of at least 1, and returning `min(requested ?? rebuildBatchCeiling(readModel), rebuildBatchCeiling(readModel))`, so a batch size of 0, a negative one and a fraction are refused and a larger one is lowered to the ceiling (E-40, F13) - fnBackfillBatch: `export const backfillBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9, E-40) - fnVerifyBatch: `export const verifyBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9) - fnPurgeBatch: `export const purgeBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D19, E-40) - fnEnumerateCall: `ctx.runQuery(target.list, { tenantId, includeDeleted: true, paginationOpts: { numItems: batchSize, cursor: pageCursor } })` returning the `paginator` page `{ page, isDone, continueCursor }` of DTOs with their stream versions, one call per batch; the query caps `numItems` at its byte-derived bound, so a batch never reads more than 8 MiB of stream rows however `batchSize` was set (D2, D10, F15, S4, E-24) -- step1: `backfillBatch` loads the generation row by ID and its progress row by `by_generation`, and returns without writing when the row is absent, `state` is not `building`, `phase` is not `backfill` or `fence` differs from its argument (E-40, D9) +- step1: `backfillBatch` loads the generation row by ID and its progress row by `by_generation`, and returns without writing when the row is absent, `state` is not `building`, `pass` is not `backfill` or `fence` differs from its argument (E-40, D9) - step2: it takes its tenant: the cursor's `tenantId`, or `nextTenant(ctx, null)` when the cursor is `null`; when the tenant list is empty the batch has no tenant, reads the gate as step 3 states, and goes to step 6 with the pass done (E-49, Law 11) - step3: it calls the write pause's `gateAllows(ctx, ["all", "tenant:" + tenantId], generationId)`, with `["all"]` alone when it has no tenant, and when the answer is not allowed it parks: it patches the progress row's `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt`, schedules nothing and returns, so the generation waits for `resumeGeneration` after the gate reopens; no batch calls `assertWritable`, because a batch parks its generation rather than failing its scheduled mutation with a thrown refusal (E-8, E-42, D9) - step4: it calls `list` once as `fnEnumerateCall` states, with the cursor's `pageCursor` (D10, E-40) - step5: for each DTO of the page it calls `applyProjection(ctx, target.readModel, { tenantId, dto, versions: [dto.version], mode: "backfill", generations: [thisGeneration] })` for this generation alone, adding one to `rowsWritten` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` for `skipped-newer` or `unchanged` (D9, E-9) -- step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"`, patches the progress row to `phase: "verify"`, `cursor: null`, `misses: 0`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) -- verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `phase` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and the progress row to `phase: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) -- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `phase` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(limitPurgeBatch)`, deletes each row and adds their number to `rowsPurged`; when it deleted `limitPurgeBatch` rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`; a pass that is not done checkpoints and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and the progress row to `phase: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) +- step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) +- verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `pass` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and the progress row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) +- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(limitPurgeBatch)`, deletes each row and adds their number to `rowsPurged`; when it deleted `limitPurgeBatch` rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`; a pass that is not done checkpoints and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) - purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) -- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `phase` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) -- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string, batchSize?: number): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `phase` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, setting the progress row's `batchSize` to `batchSizeFor(readModel, batchSize)` when one is given, clearing `lastError`, and scheduling the batch of the phase, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) -- fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain` (D9, Sc L2-5, E-40, E-48) +- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) +- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string, batchSize?: number): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, setting the progress row's `batchSize` to `batchSizeFor(readModel, batchSize)` when one is given, clearing `lastError`, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) +- fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain`, for a generation in any state whose progress row names a pass, so it continues a backfill or verify pass of a generation in `building` or `verifying` and a started purge of a `retired` or `aborted` one, which is how an operator continues a purge a restore stranded (D9, Sc L2-5, E-40, E-48) - fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) -- fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `phase` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, the in-flight error of `startGeneration` when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `phase: "verify"`, `cursor: null`, `misses: 0` and no `lastError`, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) -- fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and `"Generation " + n + " of " + name + " is " + state + "; abortGeneration needs building, verifying or verified"` unless the state is one of the three; otherwise patching the row to `state: "aborted"`, fence plus 1, `changedAt` and `changedBy`, patching the progress row to `phase: "idle"` and `lastError: "aborted: " + reason`, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8, E-48) -- fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged"` when `phase` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `phase: "purge"`, `cursor: null` and no `lastError`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) +- fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `pass` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, the in-flight error of `startGeneration` when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0` and no `lastError`, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) +- fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and `"Generation " + n + " of " + name + " is " + state + "; abortGeneration needs building, verifying or verified"` unless the state is one of the three; otherwise patching the row to `state: "aborted"`, fence plus 1, `changedAt` and `changedBy`, patching the progress row to `pass: "idle"` and `lastError: "aborted: " + reason`, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8, E-48) +- fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged; resumeGeneration continues it"` when `pass` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `pass: "purge"`, `cursor: null` and no `lastError`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) - fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(v.object({ generation: generationDoc, progress: progressDoc })), handler })` the newest `limitGenerationsListed` generations of the read model by `by_read_model` in descending order, each with its progress row, where `generationDoc` and `progressDoc` are the two tables' document validators with `_id` and `_creationTime`; an operator entry that only reads, so it takes no operator (E-8, E-40, E-48, D19) - errorUnknownGeneration: every entry and every helper that takes a `generationId` throws `"No generation " + generationId` when the row is absent, and a batch returns without writing instead (E-8) -- stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its phase is not `idle`, and the operator answers with `resumeGeneration` and a smaller batch, down to 1 (E-40, F1, F9, Probe 6) +- stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its pass is not `idle`, and the operator answers with `resumeGeneration` and a smaller batch, down to 1 (E-40, F1, F9, Probe 6) - occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch; a batch reads the generation row and never writes it outside the end of a pass, and it writes the progress row, which no command reads, so a batch's checkpoint re-runs no command (F1, D9, Probe 6, E-40) - limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) - limitPurgeBatch: `min(500, floor(4194304 / rowBudgetBytes))` rows deleted per purge batch, 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) diff --git a/design/specs/application/restore.sdp.md b/design/specs/application/restore.sdp.md index abfb693..d69fd6a 100644 --- a/design/specs/application/restore.sdp.md +++ b/design/specs/application/restore.sdp.md @@ -91,7 +91,7 @@ The restore itself is a platform operation outside any mutation. Everything the - transactionBoundary: none for the snapshot restore; each check batch is one internal mutation that reads through one component query and writes only the run record; `finishRestoreRun` is one mutation that marks the run and reschedules the stranded generations together, so acceptance and the rescheduling commit or fail as one (D19, F8, F13) - convexSurface: `startRestoreRun`, `streamCheckBatch`, `readModelCheckBatch`, `receiptCheckBatch`, `pendingWorkBatch`, `finishRestoreRun` as internal mutations and `getRestoreRun` as an internal query, all in the parent (E-42, D19) -- tableRestoreRuns: `restoreRuns: defineTable({ backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), state: v.union(v.literal("running"), v.literal("accepted"), v.literal("failed")), checks: v.array(v.object({ name: v.string(), cursor: v.optional(v.string()), examined: v.number(), findings: v.array(v.object({ subject: v.string(), detail: v.string() })), state: v.union(v.literal("pending"), v.literal("running"), v.literal("passed"), v.literal("failed")) })), pendingWork: v.array(pendingWorkItemValidator), operator: v.string(), startedAt: v.number(), finishedAt: v.optional(v.number()) })` where `checks` holds five entries, the four checks and the inventory under the name `pendingWork`, whose cursor and state are kept like a check's while its items go to `pendingWork` (E-42, D19) +- tableRestoreRuns: `restoreRuns: defineTable({ backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), state: v.union(v.literal("running"), v.literal("accepted"), v.literal("failed")), checks: v.array(v.object({ name: v.string(), cursor: v.optional(v.string()), examined: v.number(), findings: v.array(v.object({ subject: v.string(), detail: v.string() })), state: v.union(v.literal("pending"), v.literal("running"), v.literal("passed"), v.literal("failed")) })), pendingWork: v.array(pendingWorkItemValidator), operator: v.string(), startedAt: v.number(), finishedAt: v.optional(v.number()), finishedBy: v.optional(v.string()) })` where `checks` holds five entries, the four checks and the inventory under the name `pendingWork`, whose cursor and state are kept like a check's while its items go to `pendingWork` (E-42, D19) - validatorPendingWorkItem: `const pendingWorkItemValidator = v.object({ kind: v.union(v.literal("generation"), v.literal("closedScope"), v.literal("import")), subject: v.string(), detail: v.string(), generationId: v.optional(v.id("generations")), fence: v.optional(v.number()), resolution: v.optional(v.union(v.literal("resumed"), v.literal("aborted"), v.literal("kept"), v.literal("recorded"))) })` where `subject` is the read model and generation number, the scope key, or the tenant and import ID, `fence` is the generation's fence at inventory time, and `resolution` is written by `finishRestoreRun` (E-42, D19) - validatorCheckScope: `const checkScopeValidator = v.union(v.object({ kind: v.literal("tenant"), tenantId: v.string() }), v.object({ kind: v.literal("all") }))` naming which tenants a check covers; under `all` a check batch iterates the tenant list through `nextTenant`, one tenant at a time, as the rebuild's `step2` and `step6` do, one tenant named on every context call, and never calls a context without a tenant (E-42, D11, Law 11) - indexRestoreRunsByStarted: `.index("by_started", ["startedAt"])` (E-42) @@ -101,7 +101,7 @@ The restore itself is a platform operation outside any mutation. Everything the - fnReadModelCheckBatch: `export const readModelCheckBatch = internalMutation({ args: { runId: v.id("restoreRuns"), readModel: v.string(), scope: checkScopeValidator }, returns: v.boolean(), handler })` reusing the rebuild's verify logic in compare mode against the active generation (E-42, D8, D9) - fnReceiptCheckBatch: `export const receiptCheckBatch = internalMutation({ args: { runId: v.id("restoreRuns"), scope: checkScopeValidator }, returns: v.boolean(), handler })` reading 25 receipts by tenant from the receipt cursor and calling `ctx.runQuery(components.orders.queries.operations.byOperation, { tenantId, operationId })` per receipt for each context the receipt's versions name (E-42, D6, D19) - fnPendingWorkBatch: `export const pendingWorkBatch = internalMutation({ args: { runId: v.id("restoreRuns") }, returns: v.boolean(), handler })` which on its first call, when the inventory's cursor is empty, reads the gate document and appends one `closedScope` item per closed entry, then reads `by_read_model_state` for each read model the parent's declarations name and each of `building`, `verifying` and `verified` and appends one `generation` item per row with its state and fence, and on every call pages the `imports` table with `.paginate` in creation order, `limitPendingWorkBatch` rows at a time, appending one `import` item per row in `running`; it saves the page cursor on its `checks` entry and returns whether the paging is done (E-42, F12, D19, E-41) -- fnFinishRestoreRun: `export const finishRestoreRun = internalMutation({ args: { runId: v.id("restoreRuns"), operator: v.string() }, returns: v.null(), handler })` an operator entry of `spec:command.actor-and-scope`, which calls `assertOperator` first and passes the stated operator to `resumeChain`, marking the run failed when any check has a finding; otherwise refusing with a plain error while `MAINTENANCE_MODE` is still `restore` or the inventory has not finished, and then, for every `generation` item whose row is still in `building` or `verifying` at the inventoried fence, calling the rebuild's `resumeChain` in this mutation, which bumps the fence and schedules the next batch from the saved cursor, writing `resumed` on the item, `aborted` on one whose row is `aborted`, `kept` on a `closedScope` item still closed, `recorded` on an `import` item, and marking the run accepted with the finish time (E-42, E-48, D19, F8, F12, Sc L2-8) +- fnFinishRestoreRun: `export const finishRestoreRun = internalMutation({ args: { runId: v.id("restoreRuns"), operator: v.string() }, returns: v.null(), handler })` an operator entry of `spec:command.actor-and-scope`, which calls `assertOperator` first, writes the stated operator on the run as `finishedBy` whether it marks the run failed or accepted, and passes it to `resumeChain`, marking the run failed when any check has a finding; otherwise refusing with a plain error while `MAINTENANCE_MODE` is still `restore` or the inventory has not finished, and then, for every `generation` item whose row is still in `building` or `verifying` at the inventoried fence, calling the rebuild's `resumeChain` in this mutation, which bumps the fence and schedules the next batch from the saved cursor, writing `resumed` on the item, `aborted` on one whose row is `aborted`, `kept` on a `closedScope` item still closed, `recorded` on an `import` item, and marking the run accepted with the finish time (E-42, E-48, D19, F8, F12, Sc L2-8) - doorExemption: `startRestoreRun`, `streamCheckBatch`, `readModelCheckBatch`, `receiptCheckBatch`, `pendingWorkBatch` and `finishRestoreRun` do not read the gate through `assertWritable` or `gateAllows` and write only `restoreRuns`, except that `finishRestoreRun`'s accepted branch also patches the generation rows it resumes and their progress rows; `startRestoreRun` refuses when `MAINTENANCE_MODE` is not `restore`, and each later mutation refuses when its run is not `running`, so no operational row is written outside a restore and no business table is written by them at all (E-42, D19) - checkOrder: journal, domain, read model, receipt, then the inventory; a later check does not start while an earlier one has findings, because a broken journal makes the fold check meaningless, and the inventory runs last because it is not a check and its items never fail the run (E-42, D3) - gapRecord: the run record's `backupTakenAt` is the gap start; the procedure writes no synthetic receipt and invents no event for the gap (D19, Existing systems) diff --git a/design/specs/command/actor-and-scope.sdp.md b/design/specs/command/actor-and-scope.sdp.md index f3a9c20..f4a5bc0 100644 --- a/design/specs/command/actor-and-scope.sdp.md +++ b/design/specs/command/actor-and-scope.sdp.md @@ -89,7 +89,7 @@ This contract pins the shapes the parent establishes and passes into components: - validatorOperator: `const operatorValidator = v.string()` the `operator` argument of every operator entry that changes anything and the type of every field that records it (E-48) - fnAssertOperator: `assertOperator(operator: string): string` returning its argument, and throwing the plain `Error` `"An operator entry needs a stated operator"` when `operator.trim()` is empty and `"The stated operator is " + bytes + " bytes, above the limit of 512"` when `utf8Length(operator)` is above `limitOperatorBytes`; an operator entry calls it first, so a refused entry has read and written nothing (E-48, D19) - limitOperatorBytes: 512 bytes of UTF-8, the bound `limitCallText` of `spec:command.command-pipeline` gives an actor's `id` (E-48, E-34) -- operatorRecordedWhere: `startedBy` and `changedBy` on the generation row of `spec:application.generation-registry`, `changedBy` on a closed entry of the gate and `operator` on its audit record in `spec:application.write-pause` and `spec:operations.baseline-operations`, and `operator` on the run record of `spec:application.restore` (E-48, D19) +- operatorRecordedWhere: `startedBy` and `changedBy` on the generation row of `spec:application.generation-registry`, `changedBy` on a closed entry of the gate and `operator` on its audit record in `spec:application.write-pause` and `spec:operations.baseline-operations`, and `operator` and `finishedBy` on the run record of `spec:application.restore` (E-48, D19) ## Verification — reviewed diff --git a/design/specs/obligations/operator-operations.sdp.md b/design/specs/obligations/operator-operations.sdp.md index 528630d..98d7aaa 100644 --- a/design/specs/obligations/operator-operations.sdp.md +++ b/design/specs/obligations/operator-operations.sdp.md @@ -21,7 +21,7 @@ relations: Layer 3 · Detail: full where D13 rules; numbers provisional until the first experiment · Traces: D13, D11, D14, D19, F16, Law 5, Law 7, Law 8, Law 11, Sc L3-4, Sc L3-6, E-53. -The module ships five scoped operations: inspect, retry, reconcile, cancel and abandon. They are ordinary authorized parent functions on the obligation record, not domain commands, and every repair writes an `obligationRepairs` row in its own transaction, so the audit fails closed with the repair. An operator is an actor kind in the one authorization vocabulary, named on every call with the tenant scope, and a repair is refused before it is executed when the grant is missing. No operation can create a success without evidence. A reconcile without supplied evidence is a claimed attempt like every other dispatch: the operator mints the attempt and moves the obligation to pending, the claim mutation of `spec:effects.claim-call-settle` takes the lease, and the reconcile action's report meets the settle's fence; no operator operation schedules the reconcile action itself. +The module ships five scoped operations: inspect, retry, reconcile, cancel and abandon. They are ordinary authorized parent functions on the obligation record, not domain commands, and every repair writes an `obligationRepairs` row in its own transaction, so the audit fails closed with the repair. A repair is run by the actor the parent establishes, human or service, holding the repair grant in the tenant, named on every call with the tenant scope, and a repair is refused before it is executed when the grant is missing. No operation can create a success without evidence. A reconcile without supplied evidence is a claimed attempt like every other dispatch: the operator mints the attempt and moves the obligation to pending, the claim mutation of `spec:effects.claim-call-settle` takes the lease, and the reconcile action's report meets the settle's fence; no operator operation schedules the reconcile action itself. ## Intent @@ -33,7 +33,7 @@ The module ships five scoped operations: inspect, retry, reconcile, cancel and a ## Contract -- Every operation names the tenant scope and runs under an actor of kind `operator` established by the parent; the grant is read in the transaction before execution and before any obligation is disclosed (D11, Law 5, Law 11) +- Every operation names the tenant scope and runs under the actor the parent establishes, human or service, that holds `obligations:repair` or `obligations:inspect` in that tenant; the grant is read in the transaction before execution and before any obligation is disclosed (D11, Law 5, Law 11) - `inspect` returns one obligation with its repair history and its dispatch state read from `_scheduled_functions`, and it discloses nothing across tenants (D13, D19, F16) - `listByStatus` returns one tenant's obligations in a status, paginated by `updatedAt`, for the needs attention queue and the pending backlog (D13, Law 8) - `retry` is allowed from needs attention, and from pending when the next attempt is past due; it mints a new attempt, resets the rearm count, schedules the wrapper or the claim now by the record's `kind`, and writes a repair; from pending it is the fourth cause of the `pending` to `pending` transition in `spec:obligations.lifecycle-transitions`, and the old dispatch is fenced out by the new attempt ID (D13, Law 8) @@ -66,4 +66,4 @@ The module ships five scoped operations: inspect, retry, reconcile, cancel and a - errorCodeEvidenceRequired: thrown by reconcile when supplied evidence is not of kind `provider` (D13, Law 7) - errorCodeReconcileUnsupported: thrown by reconcile without evidence when the obligation's `kind` is `local` or the resolved handler declares no `reconcile`; carries `kind` and `handlerKey` (D14, E-53) - transactionBoundary: one top-level authorized mutation per repair; the schedule commits with it (D13, F8) -- disclosureRule: authorization is checked before the obligation is read and again before its evidence is returned by inspect, so a revoked operator sees nothing (Law 5) +- disclosureRule: authorization is checked before the obligation is read and again before its evidence is returned by inspect, so an actor whose grant is revoked sees nothing (Law 5) From b5ba6904ec5afb2d9156d20c7ec13c0f5231aec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:30:29 +0200 Subject: [PATCH 021/155] design: the gate's audit record names its change by kind Co-Authored-By: Claude Opus 5.5 --- design/specs/application/write-pause.sdp.md | 4 ++-- design/specs/operations/baseline-operations.sdp.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index b42f344..e5481b2 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -68,8 +68,8 @@ The gate is one document so that a command pays one point read. Scopes are strin - errorCodeWritePaused: `writePaused`, the transient code the outcome boundary reserves; the refusal's data is exactly `{ kind: "transient", code: "writePaused", message }` with no `retryAfterMs`, where `message` is `"write paused for all: restore"` under the restore door and `"write paused for tenant:" + tenantId + ": " + reason` for a closed tenant; a transient refusal, never a stored rejection, so no receipt, event, read-model row or audit record exists for the refused command (D4, Law 6, E-35) - restoreDoor: `gateAllows` runs in the parent and reads `process.env.MAINTENANCE_MODE` before the gate document; only the exact value `restore` closes the door, so the restore procedure keeps writers out even though the snapshot replaced the gate, and every rebuild batch parks until the accepted branch resumes it; the restore procedure's own six mutations call neither `gateAllows` nor `assertWritable`; no component reads the variable (E-42, D19, F11) - fnScopesOfUseCase: `scopesOfUseCase(tenantId: string, writes: readonly SourceRef[]): ScopeKey[]` returning `"all"`, the tenant scope and one source scope per `SourceRef` of the declaration's `writes`, in that order, so a command is held by a closed deployment, by its closed tenant and by a paused rebuild of any view whose sources it writes (E-8, E-38, D9) -- fnCloseScope: `closeScope(ctx: MutationCtx, scopeKey: ScopeKey, reason: string, operator: string, generationId?: Id<"generations">): Promise` throwing `"The scope " + scopeKey + " is already closed"` when `closed` holds an entry with that `scopeKey` and `"The gate holds 16 closed scopes, its limit"` when it holds `limitClosedScopes` entries; otherwise appending the entry to `closed`, inserting the document when there is none, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { action: "gate.close", scopeKey, reason, generationId, operator })` in the caller's mutation; called by `closeGate` and by the rebuild's `startGeneration`, which passes the ID of the generation row it has just inserted (E-8, E-43, E-48, D9, D19) -- fnResumeScope: `resumeScope(ctx: MutationCtx, scopeKey: ScopeKey, operator: string): Promise` throwing `"The scope " + scopeKey + " is not closed"` when `closed` holds no entry with that `scopeKey`; otherwise removing the entry, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { action: "gate.resume", scopeKey, reason: entry.reason, generationId: entry.generationId, operator })` in the caller's mutation, the reason and the generation being those of the entry it removed; called by `resumeGate` and by the rebuild's `abortGeneration` and `switchGeneration` (E-8, E-43, E-48, D9, D19) +- fnCloseScope: `closeScope(ctx: MutationCtx, scopeKey: ScopeKey, reason: string, operator: string, generationId?: Id<"generations">): Promise` throwing `"The scope " + scopeKey + " is already closed"` when `closed` holds an entry with that `scopeKey` and `"The gate holds 16 closed scopes, its limit"` when it holds `limitClosedScopes` entries; otherwise appending the entry to `closed`, inserting the document when there is none, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { kind: "gate.close", scopeKey, reason, generationId, operator })` in the caller's mutation; called by `closeGate` and by the rebuild's `startGeneration`, which passes the ID of the generation row it has just inserted (E-8, E-43, E-48, D9, D19) +- fnResumeScope: `resumeScope(ctx: MutationCtx, scopeKey: ScopeKey, operator: string): Promise` throwing `"The scope " + scopeKey + " is not closed"` when `closed` holds no entry with that `scopeKey`; otherwise removing the entry, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { kind: "gate.resume", scopeKey, reason: entry.reason, generationId: entry.generationId, operator })` in the caller's mutation, the reason and the generation being those of the entry it removed; called by `resumeGate` and by the rebuild's `abortGeneration` and `switchGeneration` (E-8, E-43, E-48, D9, D19) - fnCloseGate: `export const closeGate = internalMutation({ args: { scopeKey: v.string(), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `closeScope` with no generation: it calls `assertOperator`, throws `"closeGate takes the scope all or a tenant scope, and " + scopeKey + " is neither"` unless `scopeKey` is `all` or `tenant:` followed by a tenant ID of 1 to `limitIdLength` bytes, throws `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and then calls `closeScope`; it does not read the tenant list, so a tenant may be closed before its first grant; this is also how a meaning migration's sweep is run under a pause (E-8, E-25, E-48, D9, D19) - fnResumeGate: `export const resumeGate = internalMutation({ args: { scopeKey: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `resumeScope`: it calls `assertOperator`, makes the scope check of `closeGate` with `resumeGate` in the message, so a source scope is never reopened ahead of its rebuild, and then calls `resumeScope` (E-8, E-48, D9, Sc L2-6) - fnGetGate: `export const getGate = internalQuery({ args: {}, returns: v.object({ restore: v.boolean(), closed: v.array(closedEntryValidator) }), handler })` answering `restore` true when `MAINTENANCE_MODE` is `restore` and `closed` as the document holds it, empty when there is no document (E-8, E-42, D19) diff --git a/design/specs/operations/baseline-operations.sdp.md b/design/specs/operations/baseline-operations.sdp.md index 38e2cc5..390bc28 100644 --- a/design/specs/operations/baseline-operations.sdp.md +++ b/design/specs/operations/baseline-operations.sdp.md @@ -87,9 +87,9 @@ Optimistic-concurrency retries cannot be observed from inside a mutation, becaus - typeDiagnosticRecord: `[extension] interface DiagnosticRecord { tenantId: string; operationId: string; requestKey?: string; commandType: string; actorKind: string; actorId: string; outcome: "applied" | "businessFailure" | "rejection" | "technicalFailure" | "transientRefusal"; errorCode?: string; versions: StreamVersion[]; causedBy?: string; durationMs: number; layer: number }` (E-43, D19) - fnEmitDiagnostic: `emitDiagnostic(record: DiagnosticRecord, sink: DiagnosticSink = consoleSink): void` truncating to 4 KiB and swallowing every error the sink throws (E-43, D19, Sc ALL-1) - fnWriteAudit: `writeAudit(ctx: { db: GenericDatabaseWriter }, record: AuditRecordInput): Promise>` inserting the record with `recordedAt` through `ctx.db.insert` and letting every failure propagate; `AuditDataModel` is the data model of `auditTables` (E-43, D19, Sc ALL-1) -- tableOperatorAudit: `operatorAudit: defineTable({ action: v.union(v.literal("gate.close"), v.literal("gate.resume")), scopeKey: v.string(), reason: v.string(), generationId: v.optional(v.id("generations")), operator: v.string(), recordedAt: v.number() }).index("by_scope", ["scopeKey", "recordedAt"])` in `auditTables`; operational data of the deployment like the generation registry, so its index leads with the scope and not with a tenant, the deviation from Law 11's index rule that the registry records under E-8 (E-43, E-48, E-8, D19) +- tableOperatorAudit: `operatorAudit: defineTable({ kind: v.union(v.literal("gate.close"), v.literal("gate.resume")), scopeKey: v.string(), reason: v.string(), generationId: v.optional(v.id("generations")), operator: v.string(), recordedAt: v.number() }).index("by_scope", ["scopeKey", "recordedAt"])` in `auditTables`; operational data of the deployment like the generation registry, so its index leads with the scope and not with a tenant, the deviation from Law 11's index rule that the registry records under E-8 (E-43, E-48, E-8, D19) - indexOperatorAuditByScopeUse: `getGateAudit`, who closed and who reopened one scope, newest first; no command reads the table (E-43, D19) -- fnWriteOperatorAudit: `writeOperatorAudit(ctx: { db: GenericDatabaseWriter }, record: OperatorAuditInput): Promise>` inserting the record with `recordedAt` at the mutation's `Date.now()` and letting every failure propagate, where `type OperatorAuditInput = { action: "gate.close" | "gate.resume"; scopeKey: string; reason: string; generationId?: Id<"generations">; operator: string }`; called by the write pause's `closeScope` and `resumeScope` and by nothing else (E-43, E-48, D19) +- fnWriteOperatorAudit: `writeOperatorAudit(ctx: { db: GenericDatabaseWriter }, record: OperatorAuditInput): Promise>` inserting the record with `recordedAt` at the mutation's `Date.now()` and letting every failure propagate, where `type OperatorAuditInput = { kind: "gate.close" | "gate.resume"; scopeKey: string; reason: string; generationId?: Id<"generations">; operator: string }`; called by the write pause's `closeScope` and `resumeScope` and by nothing else (E-43, E-48, D19) - faultInjection: `DiagnosticSink` and the audit table name are injectable in the fixture app only, so the acceptance test can break each subsystem; the production composition has no injection point; the gate's audit is broken without an injection point, by a `convex-test` schema whose `operatorAudit` table has a validator that refuses the record (Sc ALL-1, Acceptance scenarios, E-43) - tablePersonalData: `personalData: defineTable({ tenantId: v.string(), ref: v.string(), subject: v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() }), fields: v.any(), createdAt: v.number(), deletedAt: v.optional(v.number()) }).index("by_ref", ["tenantId", "ref"]).index("by_subject", ["tenantId", "subject.contextId", "subject.streamType", "subject.streamId"])` (E-43, D19, Law 11) - personalDataReference: an event payload that would hold personal data rebuild does not need holds `personalRef` instead; a deletion sets `deletedAt` and clears `fields`, and the retention policy states which claims the event still makes afterward (E-43, D19) From 6407354830ece4bda3538a6c65097de3221281ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:31:40 +0200 Subject: [PATCH 022/155] design: the measured numbers on the rows of the component call, the largest order and the documents a command reads Co-Authored-By: Claude Opus 5.5 --- design/decisions/register.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/design/decisions/register.json b/design/decisions/register.json index ecacbdb..ee26cdd 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -2479,7 +2479,8 @@ "reason": "Rule 1: OQ1 is the doc's open question and D2 assumes a component per context [design/specs/context/context-component.sdp.md:49; design/PLAN.md:689]; answering it edits the doc.", "lean": { "do": "Keep a component per context: Probe 3 read a component call at about 2 ms over a helper call on a local backend and no read budget is shown broken [design/specs/context/context-component.sdp.md:41].", - "settles": "S3's measurement of one component call on the native backend, with Probe 3's quota half on a hosted deployment (OD-036), then the owner's choice." + "settles": "S3's measurement of one component call on the native backend, with Probe 3's quota half on a hosted deployment (OD-036), then the owner's choice.", + "restsOn": "Measured at the native backend tier on the pinned local backend, with other runs on the machine: a fixture component call cost 5.76 ms and 3.56 ms over a helper call in two readings of `npm run measure:place-order`, and F19's Probe 11 read 7.6 to 15.5 ms per empty component call; the backend's function log does not expose a component call inside a mutation. The quiet-machine reading is taken with `npm run measure:place-order -- --quiet`." }, "checkedBy": null, "fork": null, @@ -2534,7 +2535,8 @@ "reason": "Rule 1, product direction: the doc calls the largest order a product decision [design/specs/application/first-experiment.sdp.md:60,109].", "lean": { "do": "Measure at the provisional 100 and hand the owner the per-line counts, with the design's own bound of 256 streams per call at the 16 KiB stock budget as the ceiling the owner chooses under [design/specs/application/first-experiment.sdp.md:112; design/README.md:204].", - "settles": "S3's measurement at 1, 10 and 100 lines on the native backend, reported per line, then the owner's choice; the owner may set it before the run, which STATE.md asks for, a pointer [design/STATE.md:139]." + "settles": "S3's measurement at 1, 10 and 100 lines on the native backend, reported per line, then the owner's choice; the owner may set it before the run, which STATE.md asks for, a pointer [design/STATE.md:139].", + "restsOn": "Measured at the native backend tier on the pinned local backend: a PlaceOrder of 1, 10 and 100 lines read 8, 35 and 305 documents and wrote 6, 24 and 204, reading 2,943, 14,454 and 129,564 bytes and writing 4,271, 18,410 and 159,800; the worst contention cell, 100 lines at 32 callers on one stock item, read 9,915 documents and 4,195,493 bytes for one applied order." }, "checkedBy": null, "fork": "design/decisions/forks/OD-064.md", @@ -2613,7 +2615,7 @@ "reason": "Rule 3: what a command reads and whether the command path changes for it, settled by a measurement on the native backend [design/specs/application/first-experiment.sdp.md:63].", "lean": { "do": "Attribute the 2N reads on the native backend before any change to the command path, starting with the journal's reads per stream.", - "restsOn": "The design expected N + 6 reads and the pinned backend counted 3N + 5 at N of 1, 10 and 100 [design/specs/application/first-experiment.sdp.md:110-111]. As written, load reads the stream row through by_identity [src/context/journal.ts:82-88] and append reads up to two events of the stream's range from the expected version [src/context/journal.ts:159-168], so the code reads up to three documents per stream before it writes. **Judgment.** Those three per stream match the 3N, if the engine counts two events at a stream with one; which it counts is what the run attributes. The range read is also what serializes concurrent appends [src/context/journal.ts:157-158], so a change there changes contention, which is why attribution comes first.", + "restsOn": "The design expected N + 6 reads and the pinned backend counted 3N + 5 at N of 1, 10 and 100 [design/specs/application/first-experiment.sdp.md:110-111]. As written, load reads the stream row through by_identity [src/context/journal.ts:82-88] and append reads up to two events of the stream's range from the expected version [src/context/journal.ts:159-168], so the code reads up to three documents per stream before it writes. **Judgment.** Those three per stream match the 3N, if the engine counts two events at a stream with one; which it counts is what the run attributes. The range read is also what serializes concurrent appends [src/context/journal.ts:157-158], so a change there changes contention, which is why attribution comes first. Measured at the native backend tier on the pinned local backend: on the ten-line order the journal tail reads are 10 documents and 6,010 bytes, 29 percent of the 35 documents read.", "against": "A per-stream read set is the price of serializing appends and of the row-and-journal agreement check; removing a read to meet an expected count trades a check for a number nobody set as a target. The expected N + 6 is a design guess with no cited source.", "settles": "S3's attribution run on the native backend, the same PlaceOrder under controlled changes as the review asked, a pointer [design/STATE.md:157], with usageStats recorded per run.", "changes": "The limitExpectedDocuments and observedPlaceOrderDocuments entries of spec:application.first-experiment; if a read goes, spec:context.journal and src/context/journal.ts. S3.", From 252c5b9cb3f3bef261d854c29fa54c5c180b350a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:32:20 +0200 Subject: [PATCH 023/155] design: the lines the lanes asked of each other's files: how reruns are read, the contention example in its pack, the refused execution's record, the fourth cost target's count, a page's byte reading and the settled OQ3 row Co-Authored-By: Claude Opus 5.5 --- design/README.md | 6 +++--- design/specs/application.pack.sdp.md | 3 ++- design/specs/application/first-experiment.sdp.md | 2 +- .../constraints/one-public-execution-per-intent.sdp.md | 4 ++-- design/specs/context/queries.sdp.md | 2 +- design/specs/platform/native-harness.sdp.md | 2 +- src/context/queries.ts | 3 ++- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/design/README.md b/design/README.md index ca0281f..ca77d7a 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -217 specs · 5 packs · 115 anchors → 337 nodes · 1160 edges (0 errors, 0 warnings) +217 specs · 5 packs · 115 anchors → 337 nodes · 1161 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -246,7 +246,7 @@ The doc's own open questions: |---|---|---|---| | OQ1 | May a trivial context with no invariants of its own be plain tables in the parent behind lint rules? | A component per context everywhere; the first experiment measures component call cost (Probe 3) and decides | [`context-component`](specs/context/context-component.sdp.md), [`first-experiment`](specs/application/first-experiment.sdp.md), [`obligation-module`](specs/obligations/obligation-module.sdp.md), [`d02`](specs/decisions/d02-context-owns-state-and-journal.sdp.md) | | OQ2 | Does the product need a record of refused commands, for security audit or agent proposals? | No command uses the generic internal dispatcher; it is carried as a conditional design | [`outcome-boundary`](specs/command/outcome-boundary.sdp.md), [`agent-runs`](specs/agents/agent-runs.sdp.md), [`d07`](specs/decisions/d07-rejections-thrown-not-stored.sdp.md) | -| OQ3 | One document per stream as the default, and the largest order the placement command supports | One document per stream under a 256 KiB budget (E-2); the maximum is `MAX_ORDER_LINES`, measured at 100 | [`state-document-mapping`](specs/kernel/state-document-mapping.sdp.md), [`batch-shaped-api`](specs/context/batch-shaped-api.sdp.md), [`parent-use-cases`](specs/application/parent-use-cases.sdp.md), [`first-experiment`](specs/application/first-experiment.sdp.md), [`d10`](specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md) | +| OQ3 | One document per stream as the default, and the largest order the placement command supports | Settled: one document per stream under a 256 KiB budget (E-2) stays the default, with the derived mapping above it; the largest order `PlaceOrder` supports is 100 lines, the bound its declaration carries | [`state-document-mapping`](specs/kernel/state-document-mapping.sdp.md), [`batch-shaped-api`](specs/context/batch-shaped-api.sdp.md), [`parent-use-cases`](specs/application/parent-use-cases.sdp.md), [`first-experiment`](specs/application/first-experiment.sdp.md), [`d10`](specs/decisions/d10-contexts-meet-in-parent-use-cases.sdp.md) | | OQ4 | Where the probe app and the experiment live | Ruled by the owner on 2026-10-01: in this repository, beside the design | [`first-experiment`](specs/application/first-experiment.sdp.md), [`probe-plan`](specs/facts/probe-plan.sdp.md) | | OQ5 | Whether the decisions move into SDP carriers | This corpus is that move; facts as constraint Specs with an evidence-status target | [`transactional-domain-platform`](specs/platform/transactional-domain-platform.sdp.md) | | OQ6 | The vocabulary clashes on receipt and generation | Receipt is the stored idempotency outcome; the row that proves an effect happened is completion evidence; generation is a read-model build; the attempt number is the attempt | [`vocabulary`](specs/platform/vocabulary.sdp.md) | @@ -381,7 +381,7 @@ Cross-package fixes made in review round 2: Things in the plan that the packages found wrong, and the convention adopted: -- `` (plan 6.3) was refused by the carrier's HTML guard when the corpus was written; the placeholder is written in prose as `MAX_ORDER_LINES`, a product decision under OQ3. +- `` (plan 6.3) was refused by the carrier's HTML guard when the corpus was written; the placeholder was written in prose as `MAX_ORDER_LINES`, a product decision under OQ3; no Spec writes it since the largest order became the 100 lines `PlaceOrder`'s declaration carries. - A `workflow` Spec needs rules or examples to reach `defined`; every workflow carries `- rule:` bullets under `## Workflow`. - Unnumbered sections of the doc are cited by name (the last row of the numbering key); the plan's section 1 should list them. - The composition helper returns two static exports per command, not one; a public mutation cannot take a server-assigned namespace without letting a client claim it (Sc L1-7). diff --git a/design/specs/application.pack.sdp.md b/design/specs/application.pack.sdp.md index 2dfb82c..2ccfffb 100644 --- a/design/specs/application.pack.sdp.md +++ b/design/specs/application.pack.sdp.md @@ -26,6 +26,7 @@ specs: - spec:application.first-experiment.order-of-10-lines - spec:application.first-experiment.order-of-max-lines - spec:application.first-experiment.native-acceptance-production-configuration + - spec:application.first-experiment.native-contention - spec:application.orders-inventory-example - spec:application.orders-inventory-example.cancel-placed-order - spec:application.orders-inventory-example.cancel-twice-rejected @@ -52,4 +53,4 @@ modelRefs: Package D. Layer 2: parent use cases, read models and the projection contract, online rebuild with the generation registry and the write pause, restore, baseline operations, the cost-target constraints and the first experiment, with the Layer 2 scenarios and the all-layers audit scenario. -Membership order follows the plan's inventory with each parent's examples directly after it: parent use cases, read models, the projection contract, rebuild with its registry and gate, restore, the first experiment with the example domain, baseline operations, then the eight constraints. Forty-three Specs: nine application Specs, one operations Spec, eight constraints and twenty-five examples. Three examples are siblings the plan's table did not list, `second-context-throws`, `committed-state-read-through-query` and `write-pause-rebuild-abort`, added because each doc row enumerates two cases. Five examples verify no doc row: `first-activation-makes-read-model-writable` and `command-without-generation-fails` verify the generation registry's first activation and its failure under E-8, `query-refused-before-disclosure` and `query-refused-without-grant` verify Law 5 for a parent query, and `list-pages-by-cursor` verifies a parent list under E-24. Six more verify no doc row: the `cancel-` examples of the example domain verify `CancelOrder` under E-46, the second lifecycle command, its rejections, its receipt and its grant. +Membership order follows the plan's inventory with each parent's examples directly after it: parent use cases, read models, the projection contract, rebuild with its registry and gate, restore, the first experiment with the example domain, baseline operations, then the eight constraints. Forty-four Specs: nine application Specs, one operations Spec, eight constraints and twenty-six examples. Three examples are siblings the plan's table did not list, `second-context-throws`, `committed-state-read-through-query` and `write-pause-rebuild-abort`, added because each doc row enumerates two cases. Five examples verify no doc row: `first-activation-makes-read-model-writable` and `command-without-generation-fails` verify the generation registry's first activation and its failure under E-8, `query-refused-before-disclosure` and `query-refused-without-grant` verify Law 5 for a parent query, and `list-pages-by-cursor` verifies a parent list under E-24. Six more verify no doc row: the `cancel-` examples of the example domain verify `CancelOrder` under E-46, the second lifecycle command, its rejections, its receipt and its grant. diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 871a81f..11ed69c 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -81,7 +81,7 @@ The six cost targets are the constraint Specs this workflow is constrained by. T - rule: The kernel's fixture app is separate from the example app, and test-only functions never ship (Acceptance scenarios) - rule: [extension] The second lifecycle command is `CancelOrder` and the essential summary is the order summary read model (E-46, First experiment) - rule: The largest order `PlaceOrder` supports is 100 lines, the bound its declaration carries; it is the example's promise and no platform limit, because a command's declaration carries its own bound, chosen by its author under the adapter's stream and byte ceilings, which settles OQ3 (OQ3, D10, F13) -- rule: [extension] Each measured run produces one measurement record, and retry cost is the difference between a run with contention and the same run without it (E-47, First experiment) +- rule: [extension] Each measured run produces one measurement record; executions refused at commit, terminal executions and final outcomes are recorded separately from the backend completion records, and caller retries are recorded apart from engine reruns (E-47, First experiment) - rule: [extension] Stock exists only through `ReceiveStock`: every run, test and measurement creates its stock items by sending that command, and no function writes a stock item any other way (E-46, Law 1) - rule: [extension] A run seeds its grants by running the production composition's internal grant mutation with admin access before its first command, the read permissions `orders.read` and `inventory.read` among them, as `spec:command.actor-and-scope` pins for a tenant's first grant; the production composition names no service issuer, so every public caller is a human actor (E-37, E-15, D11) - rule: [extension] A run makes the order summary writable by running the first activation of `spec:application.generation-registry` with admin access, once, before its first `PlaceOrder` (E-8, D9) diff --git a/design/specs/constraints/one-public-execution-per-intent.sdp.md b/design/specs/constraints/one-public-execution-per-intent.sdp.md index eae615a..75e7ce3 100644 --- a/design/specs/constraints/one-public-execution-per-intent.sdp.md +++ b/design/specs/constraints/one-public-execution-per-intent.sdp.md @@ -21,7 +21,7 @@ The fourth cost target. One business intent, retried by a client, a worker or a ## Constraints -- statement: One business intent produces exactly one public command execution with one business effect and one stored outcome, across concurrent sends, lost responses and UI double submits (First experiment, D6) +- statement: One business intent produces exactly one public command execution that takes effect, with one business effect and one stored outcome, across concurrent sends, lost responses and UI double submits; a duplicate is answered from its receipt and reaches no decider, so it is no execution that takes effect (First experiment, D6) - flavor: cost - target: public-command-executions.per-business-intent.eq:1 -- measurableBy: the counts of business effects, receipts and events per intent in the idempotency scenarios of Layer 1 run inside the first experiment, and the experiment's count of executed commands per intent under contention (First experiment, Sc L1-3, Sc L1-4, E-47) +- measurableBy: the counts of business effects, receipts and events per intent in the idempotency scenarios of Layer 1 run inside the first experiment, and the experiment's count of executed commands per intent under contention; an execution counts toward the target when it takes effect, and the measurement record keeps the count of top-level executions, the original's and each duplicate's, beside it (First experiment, Sc L1-3, Sc L1-4, E-47) diff --git a/design/specs/context/queries.sdp.md b/design/specs/context/queries.sdp.md index a3189f6..81fa2fd 100644 --- a/design/specs/context/queries.sdp.md +++ b/design/specs/context/queries.sdp.md @@ -87,7 +87,7 @@ Each query is built by its own definer from the journal and a stream registratio - dtoMapping: `registration.toDto(state, meta)` is the only path from state to a DTO, so the private schema stays private; the DTO validator is the registration's and pins the shape, and every DTO carries `version: StreamVersion` built from `meta`, which the adapter's `streams` entries, the projection's version compare and the backfill all read (D8, D12, E-24) - tenantNarrowing: every index read of a query leads with the `tenantId` argument, and no query reads a row of another tenant; a DTO policy that hides fields from some actors is not designed here (Law 5, Law 11, D11, F11) - reactivityAssumption: a client subscription to the parent query re-runs when the component's tables change; this is F15, which the docs state and Probe 5 showed (F15, Probe 5) -- limitListPage: `limitListPage(budgetBytes) = max(1, min(100, floor(4 MiB / budgetBytes)))` items per page, where `budgetBytes` is the whole stream's budget: 16 at the default 256 KiB, 8 at the 512 KiB cap and 100 at 40 KiB or less; it is the cap on `numItems`, and the row cap is twice it, so a page reads at most 8 MiB of stream rows at whole budgets on the first run and on a run to an end cursor alike, and a full first page is never marked `SplitRequired` (F13, E-2, E-24, S4, Probe 5) +- limitListPage: `limitListPage(budgetBytes) = max(1, min(100, floor(4 MiB / budgetBytes)))` items per page, where `budgetBytes` is the whole stream's budget: 16 at the default 256 KiB, 8 at the 512 KiB cap and 100 at 40 KiB or less; it is the cap on `numItems`, and the row cap is twice it, so a page reads at most 8 MiB and one row at whole budgets on the first run and on a run to an end cursor alike, because the size the paginator adds is the stored row's, `_id` and `_creationTime` beside the budgeted fields, and it checks the bound after the row that reaches it, and a full first page is never marked `SplitRequired` (F13, F20, E-2, E-24, S4, Probe 5) - limitListBytes: 8 MiB, passed as `maximumBytesRead` on every `list` page; the helper stops at the row that reaches it, so one page reads at most 8 MiB and one row, half the 16 MiB read ceiling with room to spare (F13, E-2, E-24) - limitHistoryPage: 250 events per page as the cap on `numItems`, with a row cap of 500, twice the item cap, which at the 16 KiB payload bound plus the envelope reads under 9 MiB, plus the baseline events the page spans at up to the stream's `budgetBytes` each, of which a stream carries one per change of meaning; a run to an end cursor is bounded by the row cap the same way (F13, E-2, E-12, E-24) - limitHistoryBytes: 9 MiB, passed as `maximumBytesRead` on every `history` page, under the 16 MiB ceiling with room for the stream row a parent reads beside it (F13, E-12, E-24) diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index e1da46a..db6384d 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -93,7 +93,7 @@ The harness is plain test support with no Convex function in it. The entries who - sweepOwnership: [extension] the sweep reads a backend's ownership record, then the process table, then signals by process id; a process id cannot change hands while its process is alive, so the interval between the read and the signal matters only for a backend that exits during the sweep, and the harness does not close it: no call available to a process that did not spawn the backend does; a process whose command line cannot be read, such as a kernel thread that took a dead backend's process id, keeps that backend's directory and fails that sweep, and the next run starts from a fresh root (E-15) - observedWriteRate: [extension] the backend holds and then refuses writes past a rate, with `TooManyWrites` and the text "limited to 4 MiB bytes written per 1 second"; one transaction that writes 8.8 MiB commits, so the rate is not a bound on a transaction; this is why a test that writes more than the rate paces its transactions, and why a throughput measured on this backend states the limit (E-15, S13) - observedFunctionLog: [extension] the backend keeps its last 1000 function log entries in memory and loses them at a restart, and it logs one completion record per top-level call and none for a nested or component call inside it; the harness reads the log by polling `/api/stream_function_logs` from a mark built on the machine clock (E-15) -- observedOccRetry: [extension] a completion record of a mutation attempt that failed its commit on an optimistic-concurrency conflict carries `willRetry` true and an `occInfo` naming a table, and the rerun that follows in the same request has its own record; which table `occInfo` names varied between runs of one race, the receipts, events or streams table; the harness type of a completion record leaves these fields out and the tests read them from the record (E-15) +- observedOccRetry: [extension] a completion record of a mutation execution refused at commit on an optimistic-concurrency conflict carries `willRetry` true and an `occInfo` that names the table, its component path and a `retryCount` that numbers the refused executions of the request from zero, even when `error` is null; its written documents and bytes are zero, and the rerun in the same request has its own record; which table `occInfo` names varied between runs of one race, the receipts, events or streams table; the harness type of a completion record leaves these fields out and the tests read them from the record (E-15) - observedReadCount: [extension] a completion record's `usageStats.databaseReadDocuments` counts the documents the call read, and a call refused before its handler, by argument validation or by Convex's visibility rule, has a record that read none (E-15) - observedUsageStats: [extension] a top-level mutation's completion record carries `usageStats` with `databaseReadDocuments`, `databaseWriteDocuments`, `databaseReadBytes`, `databaseWriteBytes` and `databaseWriteIndexRows`, and they count what a component call inside the mutation read and wrote (E-15) - observedFailedMutationWrites: [extension] a failed top-level mutation's completion record shows no written documents, `usageStats.databaseWriteDocuments` 0, even when a component call inside it wrote before the throw, so the function log cannot show writes that rolled back (E-15) diff --git a/src/context/queries.ts b/src/context/queries.ts index 3adee95..b73cdd4 100644 --- a/src/context/queries.ts +++ b/src/context/queries.ts @@ -28,7 +28,8 @@ const contextSchema = defineSchema(contextTables); const mebibyte = 1024 * 1024; export const limitListBytes = 8 * mebibyte; // Items per list page: half of limitListBytes in whole stream budgets, so a page read to its row -// cap of twice the items stays within limitListBytes. +// cap of twice the items reads at most limitListBytes and one row: the paginator adds each stored +// row's size, _id and _creationTime included, after it has read the row. export const limitListPage = (budgetBytes: number): number => Math.max(1, Math.min(100, Math.floor((4 * mebibyte) / budgetBytes))); export type PageLimit = { items: number; bytes: number }; From a7811214d59a8531b34acc46bb6074d9792d7572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:32:40 +0200 Subject: [PATCH 024/155] test: cover migration cancellation and transaction failure --- design/README.md | 8 +- ...on-backfill.probe-6-operator-cancel.sdp.md | 31 +++++ ...ackfill.probe-6-transaction-failure.sdp.md | 31 +++++ ...migrations-fits-generation-backfill.sdp.md | 5 +- design/specs/facts/probe-plan.sdp.md | 4 +- fixture/convex/migrations.ts | 12 ++ fixture/convex/schema.ts | 1 + ...n-backfill.probe-6-operator-cancel.test.ts | 120 ++++++++++++++++++ ...ckfill.probe-6-transaction-failure.test.ts | 112 ++++++++++++++++ tests/native/migration-state.ts | 35 +++++ 10 files changed, 353 insertions(+), 6 deletions(-) create mode 100644 design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-operator-cancel.sdp.md create mode 100644 design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.sdp.md create mode 100644 tests/native/f17-migrations-fits-generation-backfill.probe-6-operator-cancel.test.ts create mode 100644 tests/native/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.test.ts create mode 100644 tests/native/migration-state.ts diff --git a/design/README.md b/design/README.md index a944410..6a9e579 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -206 specs · 5 packs · 105 anchors → 316 nodes · 1110 edges (0 errors, 0 warnings) +208 specs · 5 packs · 107 anchors → 320 nodes · 1116 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -236,7 +236,7 @@ An extension is a design claim the doc does not make. Each one is three things t ## Open questions for the owner -A `[blocking]` question in this corpus is one of two things. A deferral names the trigger that brings a Layer 4 to 6 Spec back. An assumed fact names the probe that would settle it; F17 is probed by the four native Probe 6 examples. Every other open question is `[non-blocking]`, which means the design took a provisional reading and says so where it did. The rulings that matter most, grouped by where they come from. +A `[blocking]` question in this corpus is one of two things. A deferral names the trigger that brings a Layer 4 to 6 Spec back. An assumed fact names the probe that would settle it; F17 is probed by the six native Probe 6 examples. Every other open question is `[non-blocking]`, which means the design took a provisional reading and says so where it did. The rulings that matter most, grouped by where they come from. The doc's own open questions: @@ -320,8 +320,10 @@ The Layer 2 design added two more for the same reason, and nine examples that ve | none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume` | Probe 6: a migration resumes after interruption | | none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-context-enumeration` | Probe 6: migration table scope and context enumeration | | none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor` | Probe 6: a parent batch returns a context cursor to the migrations driver | +| none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-operator-cancel` | Probe 6: an operator cancels a committed successor | +| none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-transaction-failure` | Probe 6: a transaction failure preserves the saved cursor | -Package A therefore has 83 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. +Package A therefore has 85 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. ## Integration notes diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-operator-cancel.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-operator-cancel.sdp.md new file mode 100644 index 0000000..5f6724c --- /dev/null +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-operator-cancel.sdp.md @@ -0,0 +1,31 @@ +--- +id: spec:facts.f17-migrations-fits-generation-backfill.probe-6-operator-cancel +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f17-migrations-fits-generation-backfill + verifies: spec:facts.f17-migrations-fits-generation-backfill +--- +# Probe 6: an operator cancels a committed successor + +Probe 6 · native backend tier · fixture composition. + +Native backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6. + +## Intent + +- outcome: Cancellation in a separate transaction stops a pending successor, preserves committed progress and permits resume for a parent table and a context enumeration. (Probe 6, F17) + +```gwt +Given {rows: 24} depot documents and parent migrations capped at {batch: 2} source rows +When an operator observes each committed successor pending and cancels it in a separate mutation +Then both drivers report {state: "canceled"}, commit {laterBatches: 0} later batches and resume to {copies: 1} visit per source +``` + +## Verification — executable + +- The bound values are expectations written before the first native run on the pinned releases. +- The cancel transaction must follow an admin read of a committed pending successor; a missing pending boundary or a migration already finished fails the example. +- The test compares the component row, scheduler state and parent rows after cancel and after a quiet interval, and compares the context generation cursor with the component cursor. Progress before cancel is a recorded size. +- Hosted scheduling, sustained contention and timeout exhaustion are not answered by this local example. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.sdp.md new file mode 100644 index 0000000..e523ac6 --- /dev/null +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.sdp.md @@ -0,0 +1,31 @@ +--- +id: spec:facts.f17-migrations-fits-generation-backfill.probe-6-transaction-failure +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f17-migrations-fits-generation-backfill + verifies: spec:facts.f17-migrations-fits-generation-backfill +--- +# Probe 6: a transaction failure preserves the saved cursor + +Probe 6 · native backend tier · fixture composition. + +Native backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6. + +## Intent + +- outcome: A scheduled batch that exhausts the read budget rolls back its parent rows, generation checkpoint and component writes; its failed scheduler entry permits resume from the preceding cursor. (Probe 6, F17) + +```gwt +Given {rows: 6} depot documents, batches of {batch: 2} and {blobs: 18} payloads of {kib: 960} KiB +When the scheduled context batch exceeds the read budget after attempting its rows and checkpoint +Then status is {state: "failed"}, the component stores {errors: 0} errors, {committed: 2} rows remain and resume visits every source {copies: 1} time +``` + +## Verification — executable + +- The bound values are expectations written before the first native run on the pinned releases. +- The payload read occurs after the first committed batch and after attempted row and checkpoint writes in its successor. The scheduler must report a failed execution with the bytes-read error; a caught callback error alone does not reach this boundary. +- The test reads the component migration and scheduled function through admin access, compares both cursors and parent rows, then removes the fault and resumes without a cursor override. +- Hosted scheduling, sustained contention and timeout exhaustion are not answered by this local example. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md index 1a5465d..ad1a680 100644 --- a/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md @@ -30,7 +30,7 @@ On the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `co - statement: `@convex-dev/migrations` 0.3.6 drives parent-table backfill and a parent internal mutation that pages a context query and returns a migration result; `define` alone does not enumerate another component's table. (F17, Probe 6) - flavor: convex-fact - target: evidence.status:probed -- measurableBy: Four Probe 6 example Specs bind exact steps to native tests on the pinned releases: the row race, interruption and resume, table scope and context enumeration, and the parent batch returning a context cursor. (F17, D9) +- measurableBy: Six Probe 6 example Specs bind exact steps to native tests on the pinned releases: the row race, interruption and resume, table scope and context enumeration, the parent batch returning a context cursor, operator cancellation in a separate transaction and whole-transaction failure. (F17, D9) ## Design @@ -40,3 +40,6 @@ On the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `co - checkpointEvidence: A native parent context batch writes the generation cursor and rows in its callback, then the component saves the same cursor in the enclosing mutation; throwing after the generation write rolls both parent writes back, while the component catches the error and saves failure against its unchanged cursor; a successful driver status leaves the generation `building`, since verification and activation are separate. (F17, Probe 6) - enumerationCost: The native parent context batch and the parent-table-of-pages form each call `list` once per nonempty batch and cross two component boundaries per scheduled batch that enumerates a context page, driver to parent and parent to context; the table-of-pages form has a second cursor and needs migration batch size one to bound its total source rows by one list page; a context-table callback makes one parent call per source row. (F17, Probe 6) - sourceEvidence: In `node_modules/@convex-dev/migrations/dist/client/index.js`, `define` pages its local database at lines 273 to 298, calls `migrateOne` at lines 310 to 329, and forms the batch result at lines 330 to 334 and returns it at line 369; `runOne` dispatches the supplied function handle at lines 407 to 423; in `dist/component/lib.js`, lines 79 to 86 run the batch, lines 92 to 97 schedule its successor, lines 136 to 155 catch failure, and line 157 saves state. (F17, Probe 6) + +- operatorCancelEvidence: On the pinned native backend, a separate admin cancel after observing a committed pending successor leaves both the parent-table and context-batch drivers canceled; each preserves two rows, commits no more rows during the observed interval, then resumes to twenty-four distinct summaries with one visit per source; the context generation and component cursors agree at cancel and completion. The first run with execution records shows both pending successors beginning before cancel and producing an OCC-conflict completion after cancel with zero committed writes; these attempt counts are recorded sizes, and pending does not prove that execution has not begun. (F17, Probe 6) +- transactionFailureEvidence: On the pinned native backend, a scheduled context batch reading eighteen 960 KiB payloads after attempting its rows and checkpoint fails the whole transaction with `Uncaught Error: Too many bytes read in a single function execution (limit: 16777216 bytes). Consider using smaller limits in your queries, paginating your queries, or using indexed queries with a selective index range expressions.`; the component row retains its preceding cursor, processed count two and worker ID without an error field, while the worker is failed and supplies the error; `getStatus` reports failed without an error field, both saved cursors and the two prior rows remain, and resume completes six sources once. Timeout and OCC retry exhaustion are not run. (F17, Probe 6) diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index a963de0..b0a39a3 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -20,7 +20,7 @@ relations: Feature · Detail: full transcription · Traces: Probe 1 to Probe 7, F4, F5, F12, F13, F14, F15, F16, F17, S13, E-16. -Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`, so Probe 7 need only confirm them and answer what a restore leaves; the backup page says nothing about component data or the scheduler, so that half of Probe 7 stands. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, and each result is recorded on its fact; Probe 6 runs on native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6, with its four examples and results on F17; Probe 7 has not run. +Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`, so Probe 7 need only confirm them and answer what a restore leaves; the backup page says nothing about component data or the scheduler, so that half of Probe 7 stands. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, and each result is recorded on its fact; Probe 6 runs on native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6, with its six examples and results on F17; Probe 7 has not run. The probes are not the first experiment. The experiment builds Layers 0 to 2 and measures cost; the probes settle single facts and can run before it, in a fixture app that is separate from the example app. @@ -60,7 +60,7 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - Probe 3 measures one component call from a parent mutation and from a parent query in latency and in function-call quota, and reports the difference from a plain helper; it serves D2 and D8 (Probe 3, F4) - Probe 4 reads and writes near the per-transaction limits from a parent, from a nested mutation and from a component, and records whether the limits add up across the boundaries or apply per call; it serves D10 (Probe 4, F13) - Probe 5 subscribes to a parent query that calls a component query, changes the component's data, and checks that the subscription updates; it then pages a component list built with `paginator` through the parent while live writes land inside the paged range, with the client passing the end cursor as `convex-helpers` describes, records whether the pages stay contiguous, and fills one page's range past the component's `maximumRowsRead` so that a re-run comes back `SplitRequired`, recording whether the `convex-helpers/react` hook splits it across the boundary without a gap; it serves D8 (Probe 5, F15, S4) -- Probe 6 binds four native backend examples on F17's pins: a component-driven batch racing a live depot command, cancel and resume with a thrown batch and process restart, parent and context table scope, and a parent batch returning a context list cursor to the migrations driver with its generation checkpoint in the same transaction; the results distinguish the table helper from the batch driver and serve D9. (Probe 6, F1, F17) +- Probe 6 binds six native backend examples on F17's pins: a component-driven batch racing a live depot command, cancel and resume with a thrown batch and process restart, parent and context table scope, and a parent batch returning a context list cursor to the migrations driver with its generation checkpoint in the same transaction, operator cancellation in a separate transaction and a scheduled batch failing the whole transaction on the read budget; the results distinguish the table helper from the batch driver and serve D9. (Probe 6, F1, F17) - Probe 7, before Layer 3, takes a backup with pending scheduled functions, Workpool jobs and a Workflow run in flight, restores it, and records what survives; it reads `_scheduled_functions` for a failed run and confirms the states and the retention window; it serves D13 and D19 (Probe 7, F12, F16) ## Design diff --git a/fixture/convex/migrations.ts b/fixture/convex/migrations.ts index cc805e0..e57eb58 100644 --- a/fixture/convex/migrations.ts +++ b/fixture/convex/migrations.ts @@ -108,6 +108,7 @@ export const configure = internalMutation({ failKey: v.union(v.string(), v.null()), reads: v.number(), clear: v.boolean(), + exhaustKey: v.optional(v.string()), }, handler: async (ctx, { clear, ...settings }) => { const old = await ctx.db.query("migrationSettings").unique(); @@ -226,6 +227,8 @@ export const contextBatch = internalMutation({ ) .unique())!; const settings = await ctx.db.query("migrationSettings").unique(); + for (let i = 0; i < (settings?.reads ?? 0); i++) + await ctx.db.get("generations", generation._id); for (const value of page.page) { const dto = value as Document; await writeSummary(ctx, "tenant", dto); @@ -239,6 +242,15 @@ export const contextBatch = internalMutation({ batchesDone: generation.batchesDone + 1, rowsWritten: generation.rowsWritten + page.page.length, }); + if ( + page.page.some( + (value) => (value as Document).documentId === settings?.exhaustKey, + ) + ) + await ctx.db + .query("blobs") + .withIndex("by_group", (q) => q.eq("group", "migration-budget")) + .collect(); if ( page.page.some( (value) => (value as Document).documentId === settings?.failKey, diff --git a/fixture/convex/schema.ts b/fixture/convex/schema.ts index 96867b8..a6ec54f 100644 --- a/fixture/convex/schema.ts +++ b/fixture/convex/schema.ts @@ -8,6 +8,7 @@ export default defineSchema({ migrationSettings: defineTable({ failKey: v.union(v.string(), v.null()), reads: v.number(), + exhaustKey: v.optional(v.string()), }), migrationVisits: defineTable({ key: v.string(), version: v.number() }), enumerationPages: defineTable({ diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-operator-cancel.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-operator-cancel.test.ts new file mode 100644 index 0000000..9485412 --- /dev/null +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-operator-cancel.test.ts @@ -0,0 +1,120 @@ +import { setTimeout as sleep } from "node:timers/promises"; +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe6OperatorCancelContract as contract } from "../../generated/contracts/facts.f17-migrations-fits-generation-backfill.probe-6-operator-cancel.contract.js"; +import { setup, record, untilStatus, completeRows } from "./migration-steps.js"; +import { migrationState, equalContextCursor } from "./migration-state.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f17-migrations-fits-generation-backfill.probe-6-operator-cancel", + ), + verifies: ref( + "spec:facts.f17-migrations-fits-generation-backfill.probe-6-operator-cancel", + ), +}); +void anchor; +interface World { + rows: number; + batch: number; + results: { state: string; laterBatches: number; copies: number }[]; +} +bindExample(contract, (): World => ({ rows: 0, batch: 0, results: [] }), { + "{rows} depot documents and parent migrations capped at {batch} source rows": + (world, { rows, batch }) => { + world.rows = rows; + world.batch = batch; + }, + "an operator observes each committed successor pending and cancels it in a separate mutation": + async (world) => { + for (const form of ["summaries", "contextBatch"] as const) { + const { backend } = await setup(world.rows); + const name = `migrations:${form}`; + const run = + form === "summaries" + ? "migrations:run" + : "migrations:runContextBatch"; + const mark = await backend.admin.logMark(); + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 200, + clear: false, + }); + const started = await backend.admin.run(run, { + batchSize: world.batch, + }); + const scheduled = await backend.admin.readTable( + "_scheduled_functions", + { component: "migrations" }, + ); + const pending = scheduled.filter( + (row) => (row.state as { kind: string }).kind === "pending", + ); + record(`${form} before operator cancel`, { started, pending }); + expect(pending.length).toBeGreaterThan(0); + const canceled = await backend.admin.run( + "lib:cancel", + { name }, + { component: "migrations" }, + ); + const after = await migrationState(backend, name); + record(`${form} operator cancel committed`, { + returned: canceled, + ...after, + }); + expect(after.status.state).toBe("canceled"); + expect((after.worker!.state as { kind: string }).kind).toBe("canceled"); + expect(after.status.processed).toBeGreaterThan(0); + expect(after.status.processed).toBeLessThan(world.rows); + expect(after.rows).toHaveLength(after.status.processed); + expect(after.visits).toHaveLength(after.status.processed); + if (form === "contextBatch") equalContextCursor(after); + await sleep(500); + const quiet = await migrationState(backend, name); + record(`${form} after cancel interval`, quiet); + expect(quiet).toEqual(after); + await backend.admin.run("migrations:status", {}); + const entries = await backend.admin.completionsSince(mark, (logs) => + logs.some((entry) => entry.identifier === "migrations:status"), + ); + const cancel = entries.find( + (entry) => entry.identifier === "lib:cancel", + )!; + expect(cancel).toBeDefined(); + record(`${form} cancel execution records`, { + entries, + laterCompletions: entries.filter( + (entry) => + entry.identifier === "lib:migrate" && + entry.timestamp > cancel.timestamp, + ), + }); + const laterBatches = + (quiet.visits.length - after.visits.length) / world.batch; + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 0, + clear: false, + }); + await backend.admin.run(run, { batchSize: world.batch }); + await untilStatus(backend, "success", "migrations", name); + const final = await migrationState(backend, name); + record(`${form} operator resume complete`, final); + if (form === "contextBatch") equalContextCursor(final); + const complete = await completeRows(backend, world.rows, 1); + world.results.push({ + state: after.status.state, + laterBatches, + copies: complete.visits.length / world.rows, + }); + } + }, + "both drivers report {state}, commit {laterBatches} later batches and resume to {copies} visit per source": + (world, expected) => { + expect(world.results).toEqual([expected, expected]); + }, +}); diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.test.ts new file mode 100644 index 0000000..e471ffc --- /dev/null +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.test.ts @@ -0,0 +1,112 @@ +import { expect } from "vitest"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { probe6TransactionFailureContract as contract } from "../../generated/contracts/facts.f17-migrations-fits-generation-backfill.probe-6-transaction-failure.contract.js"; +import type { Backend } from "../../harness/backend.js"; +import { paceAfterWrite } from "../../harness/wait.js"; +import { setup, record, untilStatus, completeRows } from "./migration-steps.js"; +import { migrationState, equalContextCursor } from "./migration-state.js"; +const anchor = specTest({ + id: testAnchorId( + "test:facts.f17-migrations-fits-generation-backfill.probe-6-transaction-failure", + ), + verifies: ref( + "spec:facts.f17-migrations-fits-generation-backfill.probe-6-transaction-failure", + ), +}); +void anchor; +interface World { + backend?: Backend; + rows: number; + batch: number; + failed?: Awaited>; +} +bindExample(contract, (): World => ({ rows: 0, batch: 0 }), { + "{rows} depot documents, batches of {batch} and {blobs} payloads of {kib} KiB": + async (world, { rows, batch, blobs, kib }) => { + world.backend = (await setup(rows)).backend; + world.rows = rows; + world.batch = batch; + for (let i = 0; i < blobs; i++) { + await world.backend.admin.run("limits:insertBlobs", { + group: "migration-budget", + count: 1, + size: kib * 1024, + }); + await paceAfterWrite(kib * 1024 + 256); + } + record("transaction budget payload", { + blobs, + kib, + bytes: blobs * kib * 1024, + }); + }, + "the scheduled context batch exceeds the read budget after attempting its rows and checkpoint": + async (world) => { + const backend = world.backend!; + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 0, + clear: false, + exhaustKey: "doc-02", + }); + const started = await backend.admin.run("migrations:runContextBatch", { + batchSize: world.batch, + }); + record("transaction failure start", started); + await untilStatus( + backend, + "failed", + "migrations", + "migrations:contextBatch", + ); + const failed = await migrationState(backend, "migrations:contextBatch"); + record("transaction failure saved state", failed); + world.failed = failed; + expect((failed.worker!.state as { kind: string }).kind).toBe("failed"); + expect(JSON.stringify(failed.worker!.state)).toContain( + "Too many bytes read in a single function execution", + ); + equalContextCursor(failed); + expect((failed.worker!.args as { cursor: string }[])[0]!.cursor).toBe( + failed.component.cursor, + ); + expect(failed.generation.batchesDone).toBe(1); + expect(failed.generation.rowsWritten).toBe(world.batch); + expect(failed.rows.map((row) => row.key).sort()).toEqual([ + "doc-00", + "doc-01", + ]); + await backend.admin.run("migrations:configure", { + failKey: null, + reads: 0, + clear: false, + }); + await backend.admin.run("migrations:runContextBatch", { + batchSize: world.batch, + }); + await untilStatus( + backend, + "success", + "migrations", + "migrations:contextBatch", + ); + const final = await migrationState(backend, "migrations:contextBatch"); + record("transaction failure resume complete", final); + equalContextCursor(final); + }, + "status is {state}, the component stores {errors} errors, {committed} rows remain and resume visits every source {copies} time": + async (world, { state, errors, committed, copies }) => { + expect(world.failed!.status.state).toBe(state); + expect(world.failed!.component.error === undefined ? 0 : 1).toBe(errors); + expect(world.failed!.status.error).toBeUndefined(); + expect(world.failed!.rows).toHaveLength(committed); + expect(world.failed!.visits).toHaveLength(committed); + expect(world.failed!.component.processed).toBe(committed); + await completeRows(world.backend!, world.rows, copies); + }, +}); diff --git a/tests/native/migration-state.ts b/tests/native/migration-state.ts new file mode 100644 index 0000000..2039bd0 --- /dev/null +++ b/tests/native/migration-state.ts @@ -0,0 +1,35 @@ +import { expect } from "vitest"; +import type { Backend } from "../../harness/backend.js"; +import { status } from "./migration-steps.js"; + +export async function migrationState(backend: Backend, name: string) { + const component = ( + await backend.admin.readTable("migrations", { component: "migrations" }) + ).find((row) => row.name === name)!; + const scheduled = await backend.admin.readTable("_scheduled_functions", { + component: "migrations", + }); + const worker = scheduled.find((row) => row._id === component.workerId); + const generation = (await backend.admin.readTable("generations")).find( + (row) => row.generation === 2, + )!; + const rows = (await backend.admin.readTable("documentSummaries")).filter( + (row) => row.generation === 2, + ); + const visits = await backend.admin.readTable("migrationVisits"); + return { + status: await status(backend, "migrations", name), + component, + worker, + generation, + rows, + visits, + }; +} +export function equalContextCursor( + state: Awaited>, +) { + expect((state.generation.cursor as { pageCursor: string }).pageCursor).toBe( + state.component.cursor, + ); +} From 23fdd28d38e8b990add667ae11534d35b8acc688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:32:58 +0200 Subject: [PATCH 025/155] build: the largest order's time assertion named as a bound on the execution time the log reports, not the engine's cap Co-Authored-By: Claude Opus 5.5 --- .../first-experiment.order-of-max-lines.sdp.md | 2 +- tests/native/first-experiment-steps.ts | 14 ++++++++++---- .../first-experiment.order-of-max-lines.test.ts | 6 ++++-- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/design/specs/application/first-experiment.order-of-max-lines.sdp.md b/design/specs/application/first-experiment.order-of-max-lines.sdp.md index 1683345..a8f3e83 100644 --- a/design/specs/application/first-experiment.order-of-max-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-max-lines.sdp.md @@ -29,7 +29,7 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row; and it asserts that the completion record's execution time is under the 1 second mutation timeout. +- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row; and it asserts that the execution time the completion record reports is under 1 second, a bound this example sets on that reported time and not the engine's cap on completion time, because the engine's limit is on the function's own computation and nested calls draw on a separate budget. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation carrying all 100 lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, and asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. - The test then receives stock for 101 more stock items through `ReceiveStock`, sends `PlaceOrder` with one line on each, one line more than the maximum, and asserts the `operationTooLarge` rejection whose details carry 101 items against a maximum of 100, with the receipts, both contexts' stream rows and events and the order summaries unchanged. diff --git a/tests/native/first-experiment-steps.ts b/tests/native/first-experiment-steps.ts index 9b78c31..7e3a338 100644 --- a/tests/native/first-experiment-steps.ts +++ b/tests/native/first-experiment-steps.ts @@ -41,14 +41,17 @@ export const authVariables = [ export const placeOrderIdentifier = getFunctionName(api.ordering.placeOrder); const summariesIdentifier = getFunctionName(api.readModels.listOrderSummaries); // The F13 ceilings of one transaction: 16 MiB and 32,000 documents read, 16 MiB and 16,000 -// documents written, and 1 second of execution. +// documents written. export const ceilings = { databaseReadBytes: 16 * 1024 * 1024, databaseReadDocuments: 32000, databaseWriteBytes: 16 * 1024 * 1024, databaseWriteDocuments: 16000, - executionSeconds: 1, }; +// A bound these examples set on the execution time the backend's log reports for a command. It is +// not the engine's cap on completion time: the engine's limit is on the function's own computation, +// and nested calls draw on a separate budget (F19). +export const reportedExecutionSecondsBound = 1; export const usageKeys = [ "databaseReadDocuments", "databaseWriteDocuments", @@ -334,7 +337,8 @@ export const usageOf = (record: CompletionRecord): Usage => usageKeys.map((key) => [key, required(record.usageStats[key], key)]), ) as Usage; // Every row stayed inside its budget, because a row above its budget fails the command and the command -// applied; and what it read, wrote and took stayed under the transaction's ceilings. +// applied; what it read and wrote stayed under the transaction's ceilings, and the execution time +// the log reports stayed under the examples' bound. export function budgetsAre( world: ExperimentWorld, budgets: "hold" | "are exceeded", @@ -356,7 +360,9 @@ export function budgetsAre( expect(usage.databaseWriteBytes).toBeLessThanOrEqual( ceilings.databaseWriteBytes, ); - expect(placed.own.executionTime).toBeLessThan(ceilings.executionSeconds); + expect(placed.own.executionTime).toBeLessThan( + reportedExecutionSecondsBound, + ); } } // What a PlaceOrder read and wrote, recorded on the test's entry in the run's evidence record. It is diff --git a/tests/native/first-experiment.order-of-max-lines.test.ts b/tests/native/first-experiment.order-of-max-lines.test.ts index b4ebcaf..b124c39 100644 --- a/tests/native/first-experiment.order-of-max-lines.test.ts +++ b/tests/native/first-experiment.order-of-max-lines.test.ts @@ -16,7 +16,6 @@ import { application, budgetsAre, callsPerContextAre, - ceilings, commitsAre, configurationIs, orderLines, @@ -24,6 +23,7 @@ import { projectionJobsAre, receive, recordUsage, + reportedExecutionSecondsBound, tenantId, type ExperimentWorld, } from "./first-experiment-steps.js"; @@ -84,7 +84,9 @@ bindExample( const placed = required(world.placed, "the order"); expect(placed.lines).toHaveLength(100); expect(maxOrderLines).toBe(100); - expect(placed.own.executionTime).toBeLessThan(ceilings.executionSeconds); + expect(placed.own.executionTime).toBeLessThan( + reportedExecutionSecondsBound, + ); recordUsage(world); // One line more than the maximum, every line with stock on hand, is refused before anything is // stored. From 0bc5557294c75a958b035e431e7c10a470b7385e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:33:54 +0200 Subject: [PATCH 026/155] design: the first experiment installs the order summary through its first rebuild Co-Authored-By: Claude Opus 5.5 --- .../specs/application/first-experiment.order-of-1-line.sdp.md | 2 +- .../specs/application/first-experiment.order-of-10-lines.sdp.md | 2 +- .../application/first-experiment.order-of-max-lines.sdp.md | 2 +- design/specs/application/first-experiment.sdp.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/design/specs/application/first-experiment.order-of-1-line.sdp.md b/design/specs/application/first-experiment.order-of-1-line.sdp.md index 7ff43d5..1fa4d0f 100644 --- a/design/specs/application/first-experiment.order-of-1-line.sdp.md +++ b/design/specs/application/first-experiment.order-of-1-line.sdp.md @@ -29,6 +29,6 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends `PlaceOrder` with one line, reads the backend's function log from a mark before the command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for the command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. +- On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends `PlaceOrder` with one line, reads the backend's function log from a mark before the command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for the command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation for this size. - The test reads the documents and bytes read and written from the completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts six documents written: the receipt, the order's state and event, the stock item's state and event, and the summary row; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-10-lines.sdp.md b/design/specs/application/first-experiment.order-of-10-lines.sdp.md index 51c5502..bcf2c23 100644 --- a/design/specs/application/first-experiment.order-of-10-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-10-lines.sdp.md @@ -29,6 +29,6 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` with ten lines over ten stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. +- On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` with ten lines over ten stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation carrying the order and then one on the Inventory operation carrying all ten lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts that the ten-line order wrote two documents more per added line than the one-line reference, the added stock item's state and its event, so documents written grew linearly with the lines; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-max-lines.sdp.md b/design/specs/application/first-experiment.order-of-max-lines.sdp.md index a8f3e83..d7b6c90 100644 --- a/design/specs/application/first-experiment.order-of-max-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-max-lines.sdp.md @@ -29,7 +29,7 @@ And the budgets {budgets: "hold"} ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- On the production composition, after grants, the first activation and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row; and it asserts that the execution time the completion record reports is under 1 second, a bound this example sets on that reported time and not the engine's cap on completion time, because the engine's limit is on the function's own computation and nested calls draw on a separate budget. +- On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row; and it asserts that the execution time the completion record reports is under 1 second, a bound this example sets on that reported time and not the engine's cap on completion time, because the engine's limit is on the function's own computation and nested calls draw on a separate budget. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation carrying all 100 lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, and asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. - The test then receives stock for 101 more stock items through `ReceiveStock`, sends `PlaceOrder` with one line on each, one line more than the maximum, and asserts the `operationTooLarge` rejection whose details carry 101 items against a maximum of 100, with the receipts, both contexts' stream rows and events and the order summaries unchanged. diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 11ed69c..75f0b18 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -84,7 +84,7 @@ The six cost targets are the constraint Specs this workflow is constrained by. T - rule: [extension] Each measured run produces one measurement record; executions refused at commit, terminal executions and final outcomes are recorded separately from the backend completion records, and caller retries are recorded apart from engine reruns (E-47, First experiment) - rule: [extension] Stock exists only through `ReceiveStock`: every run, test and measurement creates its stock items by sending that command, and no function writes a stock item any other way (E-46, Law 1) - rule: [extension] A run seeds its grants by running the production composition's internal grant mutation with admin access before its first command, the read permissions `orders.read` and `inventory.read` among them, as `spec:command.actor-and-scope` pins for a tenant's first grant; the production composition names no service issuer, so every public caller is a human actor (E-37, E-15, D11) -- rule: [extension] A run makes the order summary writable by running the first activation of `spec:application.generation-registry` with admin access, once, before its first `PlaceOrder` (E-8, D9) +- rule: [extension] A run installs the order summary through its first rebuild of `spec:application.generation-registry`, `startGeneration`, its batches and `switchGeneration`, with admin access, once, before its first `PlaceOrder` (E-8, D9) - Build Layer 0: the Orders decider with `PlaceOrder` and `CancelOrder`, the Inventory decider with `receive`, `allocate` and `release`, each pure, with `initial`, `decide` and `evolve`, and the L0 scenarios green (First experiment, D3) - Build Layer 1: the Orders and Inventory context components mounted by the parent, the journal, the command pipeline, receipts and tenancy, with the L1 scenarios green on the native backend (First experiment) - Build Layer 2: the `PlaceOrder` use case calling Orders once and Inventory once, the `ReceiveStock` command that creates stock, the `CancelOrder` use case releasing the allocation, the order summary read model written in both, and journal inspection queries (First experiment, D10, D8, E-46) From 12d3f5b9afce700c4436ae8c2736c14269c86609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:34:29 +0200 Subject: [PATCH 027/155] design: the ledger finding for a current state that is not the fold of its events Co-Authored-By: Claude Opus 5.5 --- design/reviews/consensus-ledger.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index 21ae30c..f4f3497 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -4710,6 +4710,23 @@ "note": "S4 on paper, 2026-10-02. Found by the mechanical read of the generations and gate text; older than that text, which made verify a repairing pass. The main thread ruled it is not fixed on paper and goes to the unit that builds the restore. The build must show: at convex-test, a read-model check over an active generation with a missing row, an older row and a row of a deleted subject records three findings and writes no read-model row; a clean generation records none.", "slice": "S5", "detector": "judgement" + }, + { + "round": 3, + "id": "s3-measurement-architecture-current-state-not-checked-against-event-fold", + "lens": "architecture", + "severity": "major", + "files": [ + "specs/application/restore.sdp.md", + "specs/context/journal.sdp.md", + "specs/application/first-experiment.sdp.md" + ], + "issue": "Current state is defined as always equal to the fold of its stream's events, and nothing checks it. On the native backend, after `ReceiveStock` recorded quantity two, admin access set the stock item's current on-hand to three with the same stream version and journal tail, and a one-line `PlaceOrder` for three was applied at the healthy cost: the journal tail check proves the tail matches the version, not that the state is the fold of the events (`spec:application.first-experiment` `observedStockFoldGap`, `tests/native/place-order-measurement.test.ts`).", + "fix": "No check is added on the command path: admin access is inside D11's trust boundary, and a check of state against the fold of its events on every command is the cost the single-document mapping exists to avoid. The restore's domain check and `rebuildStream` detect a current state that is not the fold of its stream's events and report it as a finding.", + "status": "open", + "note": "S3 measurement on the native backend, 2026-10-02. The main thread ruled no command-path check and sent it to the unit that builds the restore check and `rebuildStream`. The reproduction stays as a test that states what happens. The build must show: on the native backend or at convex-test, a stock item whose current state admin access set ahead of its events is reported by the restore's domain check and by `rebuildStream` as a state that differs from its fold, while a stream whose state is its fold reports none; and the present reproduction still states what the command path does.", + "slice": "S5", + "detector": "native backend" } ], "sessions": [ From 1a496d599f5367e97e10998364834b7bcbc382e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:34:39 +0200 Subject: [PATCH 028/155] test: describe scheduler setup and actions in their bound steps --- ...de-pending-scheduled-functions.probe-7-fresh-import.sdp.md | 4 ++-- ...pending-scheduled-functions.probe-7-in-place-import.sdp.md | 4 ++-- ...uled-functions-table-shows-failed-runs.probe-7-scan.sdp.md | 4 ++-- ...e-pending-scheduled-functions.probe-7-fresh-import.test.ts | 4 ++-- ...ending-scheduled-functions.probe-7-in-place-import.test.ts | 4 ++-- ...led-functions-table-shows-failed-runs.probe-7-scan.test.ts | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md index a4d5b68..0e42950 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md @@ -16,8 +16,8 @@ Probe 7 · native backend tier · temporary copy of the production composition. - outcome: Replacement into a fresh backend restores context data without schedules. (Probe 7) ```gwt -Given a production composition has parent and context documents, stored scheduler ids and all five scheduler states -When its exported snapshot is imported with replacement into a fresh backend with its own environment +Given a production composition can store parent and context documents and typed scheduler references +When it creates documents and five scheduler states, then exports and imports the snapshot with replacement into a fresh backend with its own environment Then documents including ids and creation times in {scopes: "parent,orders,inventory"} equal the exported documents {equal: true} And the scheduler row counts are parent {parent: 0}, Orders {orders: 0} and Inventory {inventory: 0} And the destination environment is unchanged {unchanged: true} diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md index a9a06df..1e2052e 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md @@ -16,8 +16,8 @@ Probe 7 · native backend tier · temporary copy of the production composition. - outcome: Replacement in place preserves schedules that run against restored data. (Probe 7) ```gwt -Given a production composition exports business documents and all five scheduler states before more orders and schedules are created -When replacement imports the exported snapshot in place before the pending reactions are due +Given a production composition has temporary functions for scheduler references and reactions +When it exports business documents and five scheduler states, adds orders and schedules, then imports the snapshot in place before pending reactions are due Then documents including ids and creation times in {scopes: "parent,orders,inventory"} equal the exported documents {equal: true} And every scheduler row in each scope is unchanged {unchanged: true} And later business changes are gone {removed: true} and the destination environment is unchanged {environmentUnchanged: true} diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md index e229584..955fa60 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.sdp.md @@ -16,8 +16,8 @@ Probe 7 · native backend tier · temporary copy of the production composition. - outcome: A failed-function scan reads unrelated schedules. (Probe 7) ```gwt -Given a successful local reaction and a failed reaction share the scheduler with pending calls to several functions -When a query filters the system table for one function and failed state while unrelated schedules grow past the read boundary +Given temporary functions provide a local reaction, a failed reaction and unrelated scheduled calls +When a local reaction succeeds and a query filters the system table for one function and failed state while unrelated schedules grow past the read boundary Then the last accepted total is {accepted: 4091} and the first refused total is {refused: 4092} with text {refusal: "Too many reads in a single function execution (limit: 4096)"} And each successful scan reads {documents: 0} documents and {bytes: 0} bytes, adding {queriesPerRow: 1} database query per row plus {extraQueries: 1} And the table reaches {total: 32001} rows and scans at {documentCeiling: 32000} and above are refused before any document bound is measured diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts index c270dec..b583259 100644 --- a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts @@ -23,11 +23,11 @@ vi.setConfig({ testTimeout: 300000 }); type World = SchedulerWorld>>; bindExample(contract, (): World => ({}), { - "a production composition has parent and context documents, stored scheduler ids and all five scheduler states": + "a production composition can store parent and context documents and typed scheduler references": async (world) => { Object.assign(world, await schedulingBackend()); }, - "its exported snapshot is imported with replacement into a fresh backend with its own environment": + "it creates documents and five scheduler states, then exports and imports the snapshot with replacement into a fresh backend with its own environment": async (world) => { world.observation = await freshImport(world.backend!, world.directory!); }, diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts index 85635e7..3de2a60 100644 --- a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts @@ -23,11 +23,11 @@ vi.setConfig({ testTimeout: 300000 }); type World = SchedulerWorld>>; bindExample(contract, (): World => ({}), { - "a production composition exports business documents and all five scheduler states before more orders and schedules are created": + "a production composition has temporary functions for scheduler references and reactions": async (world) => { Object.assign(world, await schedulingBackend()); }, - "replacement imports the exported snapshot in place before the pending reactions are due": + "it exports business documents and five scheduler states, adds orders and schedules, then imports the snapshot in place before pending reactions are due": async (world) => { world.observation = await inPlaceImport(world.backend!, world.directory!); }, diff --git a/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts index ca8c2e4..d7c6f69 100644 --- a/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts +++ b/tests/native/f16-scheduled-functions-table-shows-failed-runs.probe-7-scan.test.ts @@ -22,11 +22,11 @@ vi.setConfig({ testTimeout: 300000 }); type World = SchedulerWorld>>; bindExample(contract, (): World => ({}), { - "a successful local reaction and a failed reaction share the scheduler with pending calls to several functions": + "temporary functions provide a local reaction, a failed reaction and unrelated scheduled calls": async (world) => { Object.assign(world, await schedulingBackend()); }, - "a query filters the system table for one function and failed state while unrelated schedules grow past the read boundary": + "a local reaction succeeds and a query filters the system table for one function and failed state while unrelated schedules grow past the read boundary": async (world) => { world.observation = await scanCase(world.backend!); }, From 665d17c12e8a4cd8fdb45dc29cc3b548b7900652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:36:15 +0200 Subject: [PATCH 029/155] design: Probe 6's fact and example prose, and the fixture's comments F17's paragraphs say what the component drives and what it cannot reach, with the undocumented batch form as a risk; the race and table-scope examples state their results once and in plain words; the probe plan's bullet says what Probe 6 runs. Co-Authored-By: Claude Opus 5.5 --- ...f01-serializable-mutations-under-occ.sdp.md | 2 +- ...backfill.probe-6-batch-races-command.sdp.md | 9 +++++---- ...ackfill.probe-6-context-batch-cursor.sdp.md | 2 +- ...backfill.probe-6-context-enumeration.sdp.md | 15 ++++++++------- ...-migrations-fits-generation-backfill.sdp.md | 18 ++++++++++-------- design/specs/facts/probe-plan.sdp.md | 2 +- fixture/convex/depot/convex.config.ts | 3 +++ fixture/convex/depot/migrations.ts | 3 +++ fixture/convex/depot/schema.ts | 2 ++ fixture/convex/depotCommands.ts | 1 + fixture/convex/migrations.ts | 5 +++++ 11 files changed, 40 insertions(+), 22 deletions(-) diff --git a/design/specs/facts/f01-serializable-mutations-under-occ.sdp.md b/design/specs/facts/f01-serializable-mutations-under-occ.sdp.md index b403c82..1154f9d 100644 --- a/design/specs/facts/f01-serializable-mutations-under-occ.sdp.md +++ b/design/specs/facts/f01-serializable-mutations-under-occ.sdp.md @@ -12,7 +12,7 @@ F1 · Status: documented · Doc status: Documented, S1 · Decisions: D1, D9. Convex runs every mutation as a serializable transaction and resolves conflicts by retrying the loser under optimistic concurrency control. The design rests on it twice: one mutation per business operation needs no lock protocol beside it, and a rebuild backfill can run online because each batch is ordered against live commands by the same mechanism. An engine retry caused by a conflict is invisible to the caller and is distinct from the logical version conflict the persistence adapter reports. -Probe 6 on the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6 races a component-driven batch with a live depot amendment in six trials: both summary generations end at source stream version 2; the first evidence records five conflict entries and five requested retries, as sizes rather than assertions, and checks execution overlap. F1 remains documented. +Probe 6, on the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6, races a backfill batch driven by the migrations component against a live command on one read-model row, with their executions overlapping: in every trial the row ends at the source's newest stream version, and the loser of a conflict is retried by the engine. F1 stays documented. ## Intent diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.sdp.md index bd53e77..76bffa7 100644 --- a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.sdp.md +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-batch-races-command.sdp.md @@ -17,7 +17,7 @@ Native backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helper ## Intent -- outcome: The race reaches overlapping mutation calls; conflict and retry counts are recorded as sizes. Sequential checks cover each commit order. Hosted contention and throughput are not answered. (Probe 6, F17) +- outcome: A batch the component drives and a live command that write one read-model row are ordered by optimistic concurrency, so neither overwrites newer data. (Probe 6, F17, F1) ```gwt Given a document source at stream version 1 with its building summary missing @@ -29,6 +29,7 @@ Then both generations end at stream version {version: 2} with the amended title - The bound values are expectations written before the first native run on the pinned releases. - Each test owns its disposable backend; failure to reach the named boundary fails the example. -- The native backend evidence on the pinned releases records five conflict entries and five requested retries across six trials; both commit orders occur, every target finishes at version 2, and overlap is checked from execution timestamps. These counts are recorded sizes, not asserted limits. -- The missing building row forces the racing batch to insert a summary; the command updates that row if the batch commits first and skips it if the command commits first, after which the batch reads the newest source. -- The missing-row race records six conflicts and six requested retries in its first native evidence; separate serial checks bind both batch-first and command-first order to the same final stream version 2. +- The building generation's row is missing when the race starts, so the batch must insert it: the command skips a building row that does not exist yet and updates it when the batch has written it, as the rebuild's rule says, and the batch reads the newest source when it commits second. +- The test requires the batch's and the command's executions to overlap in the backend's function log, and records each trial's conflicts and engine retries as sizes. On the pinned releases each trial recorded zero or one conflict, on the context's `streams` table or the parent's `documentSummaries` table. +- Two runs without overlap, batch first and then command first, end at the same stream version 2. +- What a hosted deployment does under sustained contention is not answered by this local example. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.sdp.md index 5141172..998261a 100644 --- a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.sdp.md +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.sdp.md @@ -29,4 +29,4 @@ Then resume writes {rows: 5} distinct summaries and both saved cursors agree at - The test reads the component's migrations table and the parent's generation row through admin access, checks the saved cursors at cancel and failure, and checks that a thrown batch rolls back its row writes and generation checkpoint while the component stores the error. - Each batch makes one context list call, returns at most its bound, and writes no parent enumeration table; component-call counts are sizes recorded in evidence. - Hosted quotas, throughput, backup replacement and projection coverage under changing tenants are not answered by this local example. -- The pinned native backend preserves equal saved cursors at cancel and failure, rolls the failed checkpoint and row writes back, and finishes five rows in three committed batches with no parent enumeration rows; the component reports success while the generation remains building. +- On the pinned releases the bound values held: the backend preserves equal saved cursors at cancel and failure, rolls the failed checkpoint and row writes back, and finishes five rows in three committed batches with no parent enumeration rows; the component reports success while the generation stays `building`. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md index 3f29d11..77f6c10 100644 --- a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md @@ -15,7 +15,7 @@ Native backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helper ## Intent -- outcome: The parent cannot name the context table in its typed data model. A runtime table walk stays in the parent. A depot migration can walk streams but direct parent-table writes fail; a passed parent function handle is tested separately. A parent migration over page records can call list, with two cursors and its page size bounded explicitly. Hosted quotas are not answered. (Probe 6, F17) +- outcome: A migration walks the tables of the component it is defined in: a parent migration cannot walk a context's table, a migration mounted in the context can but cannot write a parent table, and a parent migration over a parent table of page records can call the context's `list`. (Probe 6, F17) ```gwt Given {rows: 5} depot documents and migrations mounted in the parent and depot @@ -27,9 +27,10 @@ Then the parent table walk fails with {parentRows: 0} processed context rows and - The bound values are expectations written before the first native run on the pinned releases. - Each test owns its disposable backend; failure to reach the named boundary fails the example. -- The first native run on the pinned releases expected an empty successful parent walk; it instead reported `failed` with `Uncaught Error: Index streams.by_creation_time not found.` The bound parent outcome is that observed refusal. -- A parent definition without the schema option is expected to reach the backend and complete with zero processed rows, while depot still holds all five sources; this value is bound before that form is run. -- The pinned native backend completes the depot walk with five visits, the depot callback with five parent rows, and the parent table-of-pages walk with three page records for five rows; the table-of-pages form keeps two cursors and requires migration batch size one for one bounded context page per batch. -- Before its native run, a parent definition with `customRange` but no schema option is bound to fail with `You must provide your schema to use a custom range.` -- The first native attempt without a schema option also reported `failed` with `Uncaught Error: Index streams.by_creation_time not found.`, rather than the expected successful zero-row walk; that form is rebound to the observed refusal. -- The pinned native backend returns `Uncaught Error: You must provide your schema to use a custom range.` for the no-schema range definition, as bound. +- The bound values first written expected the parent walk of the context's `streams` table to succeed over zero rows; the first run on the pinned releases reported `failed` with `Uncaught Error: Index streams.by_creation_time not found.`, with and without the schema option, and the example is bound to that observed refusal. +- The parent's typed data model refuses the context's table name in the compiled tier; the fixture declares it under an expected type error so that the runtime is reached, and the compiler fails if a later release admits the name. +- A parent `customRange` without the schema option fails with `Uncaught Error: You must provide your schema to use a custom range.`, as bound before its run. +- A parent mutation that reads a context row by its ID fails with ``Invalid argument `id` for `db.get`: expected to be an Id<"streams">, got Id<"depthRows"> instead.``, and a context migration that patches a parent row by its ID fails with the same kind of error naming a context table, so an ID is read in the namespace of the component that reads it. +- The migration mounted in the context completes over the five documents and writes the parent's summaries only through a function handle the parent supplied, one parent call per source row. +- The parent migration over page records calls `list` once per batch and writes three page records for five documents at a page of two; it carries two cursors, the component's over the page records and the context's in each record. +- Hosted quotas are not answered by this local example. diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md index ad1a680..5ed6a12 100644 --- a/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md @@ -10,16 +10,19 @@ relations: F17 · Status: probed · Doc status: Assumed · Decisions: D9. -Native backend evidence is bound to `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6. A parent-table migration races a live depot amendment without overwriting the newest summary. Cancel and resume preserve the component cursor, a thrown batch rolls back its rows while the component commits a failed status, and a backend restart on the same storage resumes pending work. +Probe 6 ran on the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6, on the fixture composition's document summary. What it showed holds for those pins. -The same native backend runs a migration mounted inside depot over depot's streams. It cannot directly write a parent summary by ID; it can call a supplied parent function handle, at one parent call per source row. A parent `define` cannot name depot's table through the parent's typed data model. For context enumeration, a parent internal mutation can call the context's `list` once, write the summary rows and generation checkpoint, and return `continueCursor`, `processed` and `isDone` to `runOne` without `define`. This form uses the context cursor in both saved checkpoints and no parent enumeration table. It keeps the tested two-row batch bound; the rebuild's byte bounds remain the batch author's duty. +The component drives two kinds of batch. A migration made with `define` walks a table of the component in which it is defined, so a parent migration walks parent tables and cannot reach a context's table: named in the parent, the context's table fails to compile in the compiled tier, and forced past the compiler the walk fails with `Uncaught Error: Index streams.by_creation_time not found.` The component's driver, `runOne`, also takes a parent internal mutation written by hand that returns `continueCursor`, `processed` and `isDone`; such a mutation can call the context's `list` once, write the read-model rows and the generation row's cursor, and return the context's cursor, which the component saves in the same transaction. That is how the component drives a batch over a context's enumeration. The README documents `define` and not this form; the result type it relies on, `MigrationResult`, is exported by the package. -On the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6, the first parent-table attempt expected an empty walk but reported `failed` with `Uncaught Error: Index streams.by_creation_time not found.` The parent schema has no context table. The first attempt without a schema option also expected a successful empty walk and returned the same failed status and exact error instead. A parent definition with `customRange` and no schema option returned `Uncaught Error: You must provide your schema to use a custom range.` as expected. (Probe 6) +The component can also be mounted inside a context component, where a migration walks the context's own table, but it cannot write a parent table: a parent ID passed in is read in the context's table namespace and refused. It reaches the parent only by calling a function handle the parent supplied, one call per source row, so the context calls the parent. + +A batch that races a live command on one read-model row is ordered by optimistic concurrency, and the row ends at the source's newest stream version in either commit order. A cancel, in the starting mutation or in a separate one, keeps the committed rows and the saved cursor, and starting the migration again continues from that cursor. A batch that throws rolls back its own writes, and the component catches the error and records `failed` with it. A batch whose whole transaction fails, as one that reads past the 16 MiB limit does, leaves the component's row as it was before the batch; `getStatus` reports `failed` from the scheduled function, without the error. A backend restart on the same storage runs the pending batch. Completion of the component's run does not move the generation: it stays `building`. ## Intent - outcome: Record the native backend evidence for parent-table batches, context enumeration, interruption and checkpoint ownership, so designs resting on the migrations component distinguish its driver from its table helper. (F17, Probe 6) -- risk: In the compiled tier with these package pins and `exactOptionalPropertyTypes`, `runOne` refuses the reference generated by `define` because `_args.batchSize` includes `undefined`; the fixture adapts the function reference explicitly to reach the native backend. (F17, Probe 6) +- risk: In the compiled tier with these pins and `exactOptionalPropertyTypes`, `runOne` refuses the reference `define` generates, because its `batchSize` admits `undefined`; the fixture casts the reference to reach the native backend. (F17, Probe 6) +- risk: The batch over a context's enumeration rests on a function the package exports types for and its README does not document, so a later release of the component may change it without notice. (F17, Probe 6) ### Open questions @@ -34,12 +37,11 @@ On the native backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `co ## Design -- raceEvidence: Six native backend trials end both generations at stream version 2; the first evidence records five conflict entries and five requested retries, each conflict's zero-based `retryCount` at 0; the missing-row evidence records six conflicts and six requested retries; these counts are sizes, not asserted limits, and execution overlap is required. (F17, F1, Probe 6) +- raceEvidence: In six trials per run whose batch and command executions overlap, both generations of the row end at stream version 2 with the command's title; each trial records zero or one conflict, on the context's `streams` table or the parent's `documentSummaries` table, and the loser is retried by the engine; the counts are sizes, not asserted limits. (F17, F1, Probe 6) - interruptionEvidence: With twelve sources and batch size two, cancel leaves two processed rows and a saved cursor; a later batch throwing after a write leaves those two rows and that cursor, with `state: failed`; resume reaches twelve distinct rows and one committed visit each; a pending batch survives a process kill and restart on the same storage. (F17, Probe 6) - stateOwner: The migrations component's `migrations` table stores `name`, `cursor`, `isDone`, `workerId`, `error`, `processed`, `latestStart` and `latestEnd`; `getStatus` derives its state from that row and its `_scheduled_functions` worker, and returns a null cursor on success while the stored completion cursor is `[]`. (F17, Probe 6) - checkpointEvidence: A native parent context batch writes the generation cursor and rows in its callback, then the component saves the same cursor in the enclosing mutation; throwing after the generation write rolls both parent writes back, while the component catches the error and saves failure against its unchanged cursor; a successful driver status leaves the generation `building`, since verification and activation are separate. (F17, Probe 6) - enumerationCost: The native parent context batch and the parent-table-of-pages form each call `list` once per nonempty batch and cross two component boundaries per scheduled batch that enumerates a context page, driver to parent and parent to context; the table-of-pages form has a second cursor and needs migration batch size one to bound its total source rows by one list page; a context-table callback makes one parent call per source row. (F17, Probe 6) - sourceEvidence: In `node_modules/@convex-dev/migrations/dist/client/index.js`, `define` pages its local database at lines 273 to 298, calls `migrateOne` at lines 310 to 329, and forms the batch result at lines 330 to 334 and returns it at line 369; `runOne` dispatches the supplied function handle at lines 407 to 423; in `dist/component/lib.js`, lines 79 to 86 run the batch, lines 92 to 97 schedule its successor, lines 136 to 155 catch failure, and line 157 saves state. (F17, Probe 6) - -- operatorCancelEvidence: On the pinned native backend, a separate admin cancel after observing a committed pending successor leaves both the parent-table and context-batch drivers canceled; each preserves two rows, commits no more rows during the observed interval, then resumes to twenty-four distinct summaries with one visit per source; the context generation and component cursors agree at cancel and completion. The first run with execution records shows both pending successors beginning before cancel and producing an OCC-conflict completion after cancel with zero committed writes; these attempt counts are recorded sizes, and pending does not prove that execution has not begun. (F17, Probe 6) -- transactionFailureEvidence: On the pinned native backend, a scheduled context batch reading eighteen 960 KiB payloads after attempting its rows and checkpoint fails the whole transaction with `Uncaught Error: Too many bytes read in a single function execution (limit: 16777216 bytes). Consider using smaller limits in your queries, paginating your queries, or using indexed queries with a selective index range expressions.`; the component row retains its preceding cursor, processed count two and worker ID without an error field, while the worker is failed and supplies the error; `getStatus` reports failed without an error field, both saved cursors and the two prior rows remain, and resume completes six sources once. Timeout and OCC retry exhaustion are not run. (F17, Probe 6) +- operatorCancelEvidence: A cancel in a separate mutation, made while the next batch is pending, leaves both the parent-table and context-batch drivers canceled; each preserves two rows, commits no more rows during the observed interval, then resumes to twenty-four distinct summaries with one visit per source; the context generation and component cursors agree at cancel and completion. The execution records show each pending successor had begun before the cancel committed, lost the optimistic-concurrency race to it and committed nothing, so a `pending` scheduled function may already be running. (F17, Probe 6) +- transactionFailureEvidence: A scheduled context batch reading eighteen 960 KiB payloads after attempting its rows and checkpoint fails the whole transaction with `Uncaught Error: Too many bytes read in a single function execution (limit: 16777216 bytes). Consider using smaller limits in your queries, paginating your queries, or using indexed queries with a selective index range expressions.`; the component row retains its preceding cursor, processed count two and worker ID without an error field, while the worker is failed and supplies the error; `getStatus` reports failed without an error field, both saved cursors and the two prior rows remain, and resume completes six sources once. Timeout and OCC retry exhaustion are not run. (F17, Probe 6) diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index b0a39a3..da7932a 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -60,7 +60,7 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - Probe 3 measures one component call from a parent mutation and from a parent query in latency and in function-call quota, and reports the difference from a plain helper; it serves D2 and D8 (Probe 3, F4) - Probe 4 reads and writes near the per-transaction limits from a parent, from a nested mutation and from a component, and records whether the limits add up across the boundaries or apply per call; it serves D10 (Probe 4, F13) - Probe 5 subscribes to a parent query that calls a component query, changes the component's data, and checks that the subscription updates; it then pages a component list built with `paginator` through the parent while live writes land inside the paged range, with the client passing the end cursor as `convex-helpers` describes, records whether the pages stay contiguous, and fills one page's range past the component's `maximumRowsRead` so that a re-run comes back `SplitRequired`, recording whether the `convex-helpers/react` hook splits it across the boundary without a gap; it serves D8 (Probe 5, F15, S4) -- Probe 6 binds six native backend examples on F17's pins: a component-driven batch racing a live depot command, cancel and resume with a thrown batch and process restart, parent and context table scope, and a parent batch returning a context list cursor to the migrations driver with its generation checkpoint in the same transaction, operator cancellation in a separate transaction and a scheduled batch failing the whole transaction on the read budget; the results distinguish the table helper from the batch driver and serve D9. (Probe 6, F1, F17) +- Probe 6 runs a backfill batch through the migrations component while a live command writes the same read-model row and checks that neither overwrites newer data; it interrupts the batches by a cancel, a thrown batch, a transaction that fails as a whole and a backend restart, and checks that they resume from the saved cursor; and it checks which tables a migration defined in the parent or in a context can walk and write, and whether the component can drive a batch over a context's enumeration; it serves D9 (Probe 6, F1, F17) - Probe 7, before Layer 3, takes a backup with pending scheduled functions, Workpool jobs and a Workflow run in flight, restores it, and records what survives; it reads `_scheduled_functions` for a failed run and confirms the states and the retention window; it serves D13 and D19 (Probe 7, F12, F16) ## Design diff --git a/fixture/convex/depot/convex.config.ts b/fixture/convex/depot/convex.config.ts index ced07f4..33249ce 100644 --- a/fixture/convex/depot/convex.config.ts +++ b/fixture/convex/depot/convex.config.ts @@ -1,5 +1,8 @@ import { defineComponent } from "convex/server"; import migrations from "@convex-dev/migrations/convex.config.js"; +// The mount name is the context's contextId, the constant in streams.ts. The migrations component is +// mounted inside this fixture context to show what a migration defined in a context can walk and +// write. const depot = defineComponent("depot"); depot.use(migrations); export default depot; diff --git a/fixture/convex/depot/migrations.ts b/fixture/convex/depot/migrations.ts index aac3070..136800b 100644 --- a/fixture/convex/depot/migrations.ts +++ b/fixture/convex/depot/migrations.ts @@ -1,3 +1,6 @@ +// Migrations defined inside the depot context, over its own streams table: one that only visits +// each stream, one that tries to write a parent row by its ID, and one that calls a parent mutation +// through the function handle the parent stored with configure. import { Migrations } from "@convex-dev/migrations"; import { type FunctionHandle, diff --git a/fixture/convex/depot/schema.ts b/fixture/convex/depot/schema.ts index fafe8b7..710b69e 100644 --- a/fixture/convex/depot/schema.ts +++ b/fixture/convex/depot/schema.ts @@ -1,6 +1,8 @@ import { defineSchema, defineTable } from "convex/server"; import { contextTables } from "../../../src/context/index.js"; import { v } from "convex/values"; +// A context component's schema holds the context library's tables. This fixture context adds the two +// tables its migrations use, see migrations.ts. export default defineSchema({ ...contextTables, migrationVisits: defineTable({ streamId: v.string() }), diff --git a/fixture/convex/depotCommands.ts b/fixture/convex/depotCommands.ts index c2c6480..0921670 100644 --- a/fixture/convex/depotCommands.ts +++ b/fixture/convex/depotCommands.ts @@ -360,6 +360,7 @@ const claimStockDeclaration = stockCommand( export const claimStock = publicCommand(claimStockDeclaration); export const claimStockInternal = internalCommand(claimStockDeclaration); +// AmendDocument with the fixture's read model: step 9 updates the document's summary row. export const amendSummarizedDocument = publicCommand({ ...amendDocumentDeclaration, name: "AmendSummarizedDocument", diff --git a/fixture/convex/migrations.ts b/fixture/convex/migrations.ts index e57eb58..3a50e91 100644 --- a/fixture/convex/migrations.ts +++ b/fixture/convex/migrations.ts @@ -1,3 +1,8 @@ +// The fixture composition's migrations, driven by the migrations component mounted in the parent. +// summaries walks the parent's document summaries of generation 1 and writes generation 2 from each +// document's current state; contextBatch is a parent mutation written by hand that pages the depot's +// list and returns the context's cursor to the component; the rest show which tables a migration +// defined in the parent can reach. Generation 2 is a building generation the tests insert. import { Migrations, type MigrationStatus, From 68b3804503265c837d646317408ca4fa39531d21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:37:18 +0200 Subject: [PATCH 030/155] design: README's sentence on Probe 6 names all its cases Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/README.md b/design/README.md index 6a9e579..82d45ef 100644 --- a/design/README.md +++ b/design/README.md @@ -270,7 +270,7 @@ Beyond the plan's list, three product decisions the packages surfaced: - The late-payment policy of `StartCheckout`: reallocate, ask a person, or refund as a separate operation ([`start-checkout-example`](specs/processes/start-checkout-example.sdp.md)). - Whether an old process run is kept, migrated or blocked on a given deploy ([`workflow-processes`](specs/processes/workflow-processes.sdp.md)). -Probes 1 to 5 ran on a native backend in slice S0 on 2026-10-01. Each result is recorded on its fact and on the Specs that rest on it, and [`probe-plan`](specs/facts/probe-plan.sdp.md) says what stays open: Probe 1 (the client guarantee holds across a restart, a closed client can lose a command, and the HTTP client does not retry, D6), Probe 2 (`ConvexError` data survives a nested mutation and a component boundary, F14, D7), Probe 3 (a component call costs about 2 ms more than a helper call on a local backend, with hosted cost and quota open, F4, D2, D8), Probe 4 (transaction limits are one budget across nested calls and components, F13, D10), Probe 5 (a parent query over a component query stays reactive and pinned pages stay contiguous, and the client hook loses rows after a capped split, F15, D8). Probe 6 has native evidence on backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6 for the row race, interruption and resume, table scope and a parent batch that returns a context cursor to the driver. Probe 7 remains pending (what a restore leaves of scheduler, Workpool and Workflow state; the system table's states and retention, F12, F16, D13, D19). +Probes 1 to 5 ran on a native backend in slice S0 on 2026-10-01. Each result is recorded on its fact and on the Specs that rest on it, and [`probe-plan`](specs/facts/probe-plan.sdp.md) says what stays open: Probe 1 (the client guarantee holds across a restart, a closed client can lose a command, and the HTTP client does not retry, D6), Probe 2 (`ConvexError` data survives a nested mutation and a component boundary, F14, D7), Probe 3 (a component call costs about 2 ms more than a helper call on a local backend, with hosted cost and quota open, F4, D2, D8), Probe 4 (transaction limits are one budget across nested calls and components, F13, D10), Probe 5 (a parent query over a component query stays reactive and pinned pages stay contiguous, and the client hook loses rows after a capped split, F15, D8). Probe 6 ran on backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6 (the component drives parent-table batches and, through a parent mutation that returns the context's cursor, batches over a context's enumeration; a migration cannot walk another component's table; cancel, a thrown batch, a failed transaction and a restart resume from the saved cursor, F17, F1, D9). Probe 7 remains pending (what a restore leaves of scheduler, Workpool and Workflow state; the system table's states and retention, F12, F16, D13, D19). Every open question in the corpus, by Spec, is recipe 20 of the Protocol's catalog, the open-question register. It lists the Specs that hold a blocking question first and reports the totals, so no table is kept here: From 02e3824135e4de1b83004117d21e0ff7f6c39615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:40:01 +0200 Subject: [PATCH 031/155] build: the migrations component pinned for the fixture's probe only, as a dev dependency Co-Authored-By: Claude Opus 5.5 --- package-lock.json | 42 ++++++++++++++++++++++++++++++++++++------ package.json | 4 +--- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index f4899fd..5ff4802 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,11 +7,9 @@ "": { "name": "libar-platform", "version": "0.0.0", - "dependencies": { - "@convex-dev/migrations": "0.3.6" - }, "devDependencies": { "@convex-dev/eslint-plugin": "5.0.0", + "@convex-dev/migrations": "0.3.6", "@edge-runtime/vm": "5.0.0", "@eslint/js": "10.0.1", "@libar-dev/software-delivery-protocol": "file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz", @@ -78,6 +76,7 @@ "version": "0.3.6", "resolved": "https://registry.npmjs.org/@convex-dev/migrations/-/migrations-0.3.6.tgz", "integrity": "sha512-cURYG4uy323MDi55QWNF5HAGemS4QspSgJ4uP+gGPAJdBLBbk2KFO7toFlKhacc1NinJQiG6+D/YEFyFyBS1/A==", + "dev": true, "license": "Apache-2.0", "peerDependencies": { "convex": "^1.42.0", @@ -131,6 +130,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -147,6 +147,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -163,6 +164,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -179,6 +181,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -195,6 +198,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -211,6 +215,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -227,6 +232,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -243,6 +249,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -259,6 +266,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -275,6 +283,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -291,6 +300,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -307,6 +317,7 @@ "cpu": [ "loong64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -323,6 +334,7 @@ "cpu": [ "mips64el" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -339,6 +351,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -355,6 +368,7 @@ "cpu": [ "riscv64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -371,6 +385,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -387,6 +402,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -403,6 +419,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -419,6 +436,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -435,6 +453,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -451,6 +470,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -467,6 +487,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -483,6 +504,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -499,6 +521,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -515,6 +538,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -531,6 +555,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1927,7 +1952,7 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -2436,6 +2461,7 @@ "version": "1.46.0", "resolved": "https://registry.npmjs.org/convex/-/convex-1.46.0.tgz", "integrity": "sha512-dhBwDQTZDOhi5SUGAX17EWNehCdNq6/UFrSL0eLZuGfMkdsQbCVWVRZCzbZ5CF+/pgvoGxtApbEoTSipqL4JTw==", + "dev": true, "license": "Apache-2.0", "dependencies": { "esbuild": "0.27.0", @@ -2474,6 +2500,7 @@ "version": "0.1.124", "resolved": "https://registry.npmjs.org/convex-helpers/-/convex-helpers-0.1.124.tgz", "integrity": "sha512-Gbu/jq6I3x5Y51cXP5c7+CI3t15vX5Xi2G0tOym3rxpBtYqrI+qWwxkqZgJYrHHccY42wZudB8h6kS3fak8Aeg==", + "dev": true, "license": "Apache-2.0", "bin": { "convex-helpers": "bin.cjs" @@ -2596,6 +2623,7 @@ "version": "0.27.0", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.0.tgz", "integrity": "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==", + "dev": true, "hasInstallScript": true, "license": "MIT", "bin": { @@ -3615,6 +3643,7 @@ "version": "3.9.9", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.9.tgz", "integrity": "sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg==", + "dev": true, "license": "MIT", "bin": { "prettier": "bin/prettier.cjs" @@ -3660,7 +3689,7 @@ "version": "19.3.0", "resolved": "https://registry.npmjs.org/react/-/react-19.3.0.tgz", "integrity": "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog==", - "devOptional": true, + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -3854,7 +3883,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/@typescript/typescript6/-/typescript6-6.0.2.tgz", "integrity": "sha512-mbCddXd+jm7hfx7w2YU64/Av4/NqqeG3GoRZgxPcgoTxYjhrcfJRw9ULch71SS4G+Q3bOXFhRvPqjguN0Hyp5w==", - "devOptional": true, + "dev": true, "license": "Apache-2.0", "dependencies": { "@typescript/old": "npm:typescript@^6" @@ -4273,6 +4302,7 @@ "version": "8.21.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "dev": true, "license": "MIT", "engines": { "node": ">=10.0.0" diff --git a/package.json b/package.json index c24ea5d..8f5d56c 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ }, "devDependencies": { "@convex-dev/eslint-plugin": "5.0.0", + "@convex-dev/migrations": "0.3.6", "@edge-runtime/vm": "5.0.0", "@eslint/js": "10.0.1", "@libar-dev/software-delivery-protocol": "file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz", @@ -49,8 +50,5 @@ "typescript": "npm:@typescript/typescript6@6.0.2", "typescript-eslint": "8.71.0", "vitest": "5.0.3" - }, - "dependencies": { - "@convex-dev/migrations": "0.3.6" } } From 47d38b9a221c30b5264c9fdfc69bb2d2edf76989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:41:15 +0200 Subject: [PATCH 032/155] design: the rebuild drives every batch with its own self-scheduled mutation and uses the migrations component for nothing Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- design/specs/application/rebuild.sdp.md | 4 ++-- .../decisions/d09-rebuild-online-by-default.sdp.md | 10 +++++----- .../f17-migrations-fits-generation-backfill.sdp.md | 2 ++ design/specs/facts/probe-plan.sdp.md | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/design/README.md b/design/README.md index 7fe178b..09e298a 100644 --- a/design/README.md +++ b/design/README.md @@ -218,7 +218,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-37 | [`command.tenancy-and-authority`](specs/command/tenancy-and-authority.sdp.md) | Identity to actor mapping from `tokenIdentifier`, `subject` and `issuer`; grants written by the plain helpers `insertGrant` and `revokeGrant`, a tenant's first grant created with admin access through the composition's own internal grant mutation, with no grant function, bootstrap, first-admin flow or invitation in the library; revocation by deleting the grant row, with no cache; a 500-grant read bound | | E-38 | [`command.command-declaration`](specs/command/command-declaration.sdp.md) | The declaration also carries `contractVersion`, `retention`, `irreversible`, `admission`, `bounds`, `rejections`, `audit` and `writes`; `bounds` is the one input bound, checked at the pipeline's step 1; `writes` names the context and stream type pairs the executor may write, from which step 7 derives the write pause's source scopes and which step 9 proves; the declaration has no reactions field | | E-39 | [`command.idempotency-and-receipts`](specs/command/idempotency-and-receipts.sdp.md) | The second UI submit of a create is answered with the rejection `entityExists`, not an idempotent success, because the UI path writes no receipt to replay from and the client holds the entity ID | -| E-40 | [`application.rebuild`](specs/application/rebuild.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | A batch is one mutation with one cursor on the progress row, at most 100 entities capped by the row budget and by the context's byte-derived page bound, a design number not attributed to the component; a self-scheduled internal mutation drives the context-enumerated batches, the backfill and the verify pass, and `@convex-dev/migrations` is the driver of a parent-table batch, the purge, once Probe 6 shows it fits, the purge being self-scheduled until then; one batch names one tenant, taken from the tenant list; a fence that every interrupt, resume, rollback, abort and purge start raises | +| E-40 | [`application.rebuild`](specs/application/rebuild.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | A batch is one mutation with one cursor on the progress row, at most 100 entities capped by the row budget and by the context's byte-derived page bound, a design number not attributed to the component; a self-scheduled internal mutation drives every batch, the backfill, the verify pass and the purge, and the rebuild uses `@convex-dev/migrations` for nothing; one batch names one tenant, taken from the tenant list; a fence that every interrupt, resume, rollback, abort and purge start raises | | E-41 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | The caller-driven import command with an `imports` progress row; the input bound itself is the declaration's `bounds.maxItems` under E-38 | | E-42 | [`application.restore`](specs/application/restore.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | Four checks (journal, domain, read model, receipt) as checkpointed batches, the stream check bounded through the `rebuild` query's `maxEvents` so its worst case is one fold, and a refused fold counted as a journal finding; a fifth step that inventories the closed gate entries, the generations in flight and the running imports the snapshot restored; the `restoreRuns` record; an accepted branch that resumes every stranded generation, proven by the generation row's fence; `MAINTENANCE_MODE=restore` as the writer door, read by the write pause's check before the gate document and cleared before the durable profile's schedule rebuild runs | | E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed, with `AuditRecordInput` and the source of each field in the command; the `operatorAudit` table for the gate changes an operator entry makes; Convex's own audit logging recorded as the do-nothing option; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 820f25e..1fbdd28 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -56,7 +56,7 @@ The registry, the progress row, the per-entity marker and the gate document are ### Open questions - [non-blocking] Probe 6 pending: a backfill batch racing a live command under optimistic concurrency, and whether `@convex-dev/migrations` can drive a batch from inside or across a component boundary (Probe 6, F1, F17, D9) -- [non-blocking] Extension E-40: the doc binds resumable batching to `@convex-dev/migrations` and gives no batch size; the component iterates a parent table while the sources of a read model live in a context component. The option taken here: a batch is one mutation with one cursor on the progress row and a bound of at most 100 entities, capped by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget; a batch whose source is a context's enumeration, the backfill and the verify pass, is driven by a self-scheduled internal mutation, and a batch whose source is a parent table, the purge, is the component's to drive under the same batch contract once Probe 6 shows it fits; until then the purge is self-scheduled too, `purgeDriver` is the one sentence the probe changes, and the progress row stays the one checkpoint either way (E-40, E-24, E-9, F17, F13, F1) +- [non-blocking] Extension E-40: the doc binds resumable batching to `@convex-dev/migrations` and gives no batch size; the component iterates a parent table while the sources of a read model live in a context component. The option taken here: a batch is one mutation with one cursor on the progress row and a bound of at most 100 entities, capped by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget; every batch, the backfill and the verify pass over a context's enumeration and the purge over a parent table, is driven by the platform's own self-scheduled internal mutation, and the rebuild uses the component for nothing, because Probe 6 showed that it walks only the tables of the component it is defined in and drives a context's rows only through a parent internal mutation written by hand against a contract its README does not document; the progress row is the one checkpoint (E-40, E-24, E-9, F17, F13, F1) - [non-blocking] Extension E-8: the registry states, the rollback period, what a rollback does, the gate document, the scope a paused rebuild closes and the reach of a generation are design choices the doc does not make; they are pinned in the two contracts under this workflow (E-8, E-9, E-38, D9) - [non-blocking] Extension E-8, what this workflow does not offer: the online path is pinned for a per-entity read model with one source stream type; an `AggregateProjection` and a row built from more than one source stream type have no rebuild here, and each gets one when a view needs it; the paused path of a history view is given below in outline, its cursor across several sources and its verify pass are that view's own rules, and `rollbackGeneration` refuses a generation that needs a write pause, because nothing would hold its sources while live commands fold onto a row that missed the events since its retirement (E-8, E-9, D9, D18) @@ -79,7 +79,7 @@ The registry, the progress row, the per-entity marker and the gate document are - rule: [extension] A generation spans every tenant of the deployment: its batches take one tenant at a time from `nextTenant` of the tenant list, in tenant-ID order, and name that tenant on every context call, so no call runs with an absent tenant as a wildcard, and the one registry row per read model answers every tenant's queries (D11, Law 11, E-8, E-49) - rule: [extension] A tenant whose row enters the tenant list behind a backfill's cursor is not visited by that pass and needs no visit, because every stream it owns was created after the generation was registered and so was written live; the verify pass starts again at the first tenant and visits it (E-49, E-8, D9) - rule: A backfill batch racing a live command is ordered by optimistic concurrency; the batch's read-then-write of a row is one mutation, so the batch never overwrites a row a command wrote after the batch read its source (D9, F1) -- rule: [extension] A batch whose source is a context's enumeration is driven by a self-scheduled internal mutation, and a batch whose source is a parent table by `@convex-dev/migrations` under the same batch contract where Probe 6 shows the component fits (E-40, F17, F8) +- rule: [extension] Every batch, whether its source is a context's enumeration or a parent table, is driven by a self-scheduled internal mutation under one batch contract, and the rebuild uses `@convex-dev/migrations` for nothing (E-40, F17, F8) - rule: [extension] A projection version change is a new generation, never an in-place rewrite of the active one (E-8, D8) - rule: [extension] The batch bound is a field of the progress row, a whole number of at least 1 and at most `rebuildBatchCeiling`, and never more than the context's `list` page admits at the stream type's document budget; only `resumeGeneration` changes it (E-40, E-24, E-9, F1, F13) - rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them returns without writing (E-40, E-8, F8) diff --git a/design/specs/decisions/d09-rebuild-online-by-default.sdp.md b/design/specs/decisions/d09-rebuild-online-by-default.sdp.md index 5571f9f..72b492d 100644 --- a/design/specs/decisions/d09-rebuild-online-by-default.sdp.md +++ b/design/specs/decisions/d09-rebuild-online-by-default.sdp.md @@ -21,13 +21,13 @@ Because read models update inside commands, rebuilding a per-entity or current-s - problem: A per-entity read model is rebuilt while commands run and the rebuild is interrupted and resumed; a cross-stream history view is rebuilt under a write pause and interrupted; a live command's incremental update on a row the backfill has not written yet builds a partial row, and a backfill batch can overwrite newer data (Sc L2-5, Sc L2-6) - outcome: A per-entity or current-state read model rebuilds online as a new generation while commands run, with no stale overwrite, no missing entity and no side effects; a history-dependent view rebuilds under a write pause with resume and abort (D9) - value: The common rebuild costs no downtime, and the write pause is paid only where a consistent cut is needed (D9) -- risk: The standing cost is a generation registry, a generation on every rebuildable row, a per-entity marker for counts and sums, the migrations component, and for the write pause a maintenance gate every writer reads (D9) +- risk: The standing cost is a generation registry, a generation on every rebuildable row, a per-entity marker for counts and sums, a progress row per generation, and for the write pause a maintenance gate every writer reads (D9) - assumption: Mutations are serializable under optimistic concurrency, so each backfill batch is ordered against live commands (F1) -- assumption: `@convex-dev/migrations` fits a generation backfill (F17) +- assumption: A self-scheduled internal mutation of the parent that keeps its cursor on a progress row gives resumable batching over a context's enumeration and a parent table alike; F17 records why `@convex-dev/migrations` is not used for it (F17) ### Open questions -- [non-blocking] Probe 6 pending: a backfill batch racing a live command under optimistic concurrency, and the migrations component fitting a generation backfill (Probe 6, F1, F17, D9) +- [non-blocking] Probe 6 ran a backfill batch racing a live command and the migrations component on the native backend, with its results on F17; a hosted deployment and sustained contention are not run (Probe 6, F1, F17, D9) ## Decision @@ -39,8 +39,8 @@ Because read models update inside commands, rebuilding a per-entity or current-s - rationale: Because read models update inside commands, rebuilding a per-entity or current-state read model online is safe on Convex, and optimistic concurrency orders each batch against live commands so neither overwrites newer data (D9, D8, F1) - rationale: The write pause costs more than the online path, so it is the exception, for read models that depend on history across several streams where rebuild needs a consistent cut (D9) - consequence: An incremental update on a missing row would build a partial one, so a command on an existing stream updates the building generation's row only if backfill has already written it (D9) -- consequence: `@convex-dev/migrations` provides resumable batching (D9, F17) +- consequence: The rebuild's resumable batching is its own self-scheduled internal mutation, and `@convex-dev/migrations` is used for nothing, because Probe 6 showed that the component walks only the tables of the component it is defined in, drives a context's rows only through a parent internal mutation written by hand against a contract its README does not document, keeps a status apart from the generation's state, and leaves no error on its row when a batch fails past the read limit (D9, F17) - consequence: Counts and sums need a per-entity marker so live commands and backfill never count one entity twice, which is still transactional (D9) - consequence: Under the write pause every writer in the scope reads a maintenance gate, background writers are fenced or drained, and an operator can resume or abort (D9, Sc L2-6) - consequence: Rebuild never runs commands or external effects (D9, Law 10) -- consequence: The standing cost is a generation registry and gate, a generation on every rebuildable row, markers for counts, the migrations component and its checkpoint state, and operator duties for switch, rollback, resume and abort (D9) +- consequence: The standing cost is a generation registry and gate, a generation on every rebuildable row, markers for counts, a progress row per generation that is the one checkpoint, and operator duties for switch, rollback, resume and abort (D9) diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md index 5ed6a12..fa58786 100644 --- a/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md @@ -18,6 +18,8 @@ The component can also be mounted inside a context component, where a migration A batch that races a live command on one read-model row is ordered by optimistic concurrency, and the row ends at the source's newest stream version in either commit order. A cancel, in the starting mutation or in a separate one, keeps the committed rows and the saved cursor, and starting the migration again continues from that cursor. A batch that throws rolls back its own writes, and the component catches the error and records `failed` with it. A batch whose whole transaction fails, as one that reads past the 16 MiB limit does, leaves the component's row as it was before the batch; `getStatus` reports `failed` from the scheduled function, without the error. A backend restart on the same storage runs the pending batch. Completion of the component's run does not move the generation: it stays `building`. +So the component does not fit a generation backfill as the platform needs one: it reaches a context's rows only through a parent mutation written by hand against a contract its README does not document and the compiled tier needs a cast for, its status and the generation's state are two vocabularies, and a failure past the read limit leaves no error on its row. The rebuild drives every batch, backfill, verify and purge, with its own self-scheduled internal mutation, uses the component for nothing, and keeps the progress row as the one checkpoint (D9). + ## Intent - outcome: Record the native backend evidence for parent-table batches, context enumeration, interruption and checkpoint ownership, so designs resting on the migrations component distinguish its driver from its table helper. (F17, Probe 6) diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index 68b6ca6..a8002a2 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -65,7 +65,7 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and - Probe 3 measures one component call from a parent mutation and from a parent query in latency and in function-call quota, and reports the difference from a plain helper; it serves D2 and D8 (Probe 3, F4) - Probe 4 reads and writes near the per-transaction limits from a parent, from a nested mutation and from a component, and records whether the limits add up across the boundaries or apply per call; it serves D10 (Probe 4, F13) - Probe 5 subscribes to a parent query that calls a component query, changes the component's data, and checks that the subscription updates; it then pages a component list built with `paginator` through the parent while live writes land inside the paged range, with the client passing the end cursor as `convex-helpers` describes, records whether the pages stay contiguous, and fills one page's range past the component's `maximumRowsRead` so that a re-run comes back `SplitRequired`, recording whether the `convex-helpers/react` hook splits it across the boundary without a gap; it serves D8 (Probe 5, F15, S4) -- Probe 6 runs a backfill batch through the migrations component while a live command writes the same read-model row and checks that neither overwrites newer data; it interrupts the batches by a cancel, a thrown batch, a transaction that fails as a whole and a backend restart, and checks that they resume from the saved cursor; and it checks which tables a migration defined in the parent or in a context can walk and write, and whether the component can drive a batch over a context's enumeration; it serves D9 (Probe 6, F1, F17) +- Probe 6 runs a backfill batch through the migrations component while a live command writes the same read-model row and checks that neither overwrites newer data; it interrupts the batches by a cancel, a thrown batch, a transaction that fails as a whole and a backend restart, and checks that they resume from the saved cursor; and it checks which tables a migration defined in the parent or in a context can walk and write, and whether the component can drive a batch over a context's enumeration; it serves D9, and what it showed, a component that walks only its own tables and reaches a context only through a parent mutation written by hand, is why the rebuild drives every batch with its own self-scheduled internal mutation (Probe 6, F1, F17, D9) - Probe 7, before Layer 3, takes a backup with pending scheduled functions, Workpool jobs and a Workflow run in flight, restores it, and records what survives; it reads `_scheduled_functions` for a failed run and confirms the states and the retention window; it serves D13 and D19 (Probe 7, F12, F16) - Probe 8 writes commit timestamp placeholders in the parent and two components, compares committed rows and the runtime upper bound, and checks readback, return and scheduling behavior (Probe 8, F18) - Probe 9 repeats empty nested calls in one mutation until the backend refuses it, and compares that error with the one a loop of computation gets (Probe 9, F19) From 88a7a2cb0f73245504ca42974614b754df2ddaa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:46:13 +0200 Subject: [PATCH 033/155] build: the gate, its audit record, the operator entry and the tenant list The maintenance gate's table, gateAllows with the restore door read before the gate document, assertWritable at the pipeline's step 7 before the operation is minted, closeScope and resumeScope with their operator audit record in the same mutation, and the four operator entries re-exported by each composition's gate module. The audit tables, writeAudit at step 10 for a declaration that sets audit, and writeOperatorAudit. The operator check, the tenants table written by insertGrant, and nextTenant. Codegen for both compositions. Co-Authored-By: Claude Opus 5.5 --- example/convex/_generated/api.d.ts | 2 + example/convex/gate.ts | 9 ++ example/convex/schema.ts | 7 +- fixture/convex/_generated/api.d.ts | 2 + fixture/convex/gate.ts | 9 ++ fixture/convex/schema.ts | 7 +- src/audit/index.ts | 16 +++ src/audit/tables.ts | 77 ++++++++++++++ src/audit/write.ts | 40 ++++++++ src/command/actor-and-scope.ts | 24 ++++- src/command/authority.ts | 33 +++++- src/command/declaration.ts | 2 + src/command/index.ts | 5 + src/command/pipeline.ts | 52 +++++++++- src/command/tables.ts | 12 ++- src/gate/entries.ts | 105 +++++++++++++++++++ src/gate/gate.ts | 160 +++++++++++++++++++++++++++++ src/gate/index.ts | 29 ++++++ src/gate/tables.ts | 35 +++++++ tests/pure/command.test.ts | 3 +- 20 files changed, 615 insertions(+), 14 deletions(-) create mode 100644 example/convex/gate.ts create mode 100644 fixture/convex/gate.ts create mode 100644 src/audit/index.ts create mode 100644 src/audit/tables.ts create mode 100644 src/audit/write.ts create mode 100644 src/gate/entries.ts create mode 100644 src/gate/gate.ts create mode 100644 src/gate/index.ts create mode 100644 src/gate/tables.ts diff --git a/example/convex/_generated/api.d.ts b/example/convex/_generated/api.d.ts index 1d14f16..7df9760 100644 --- a/example/convex/_generated/api.d.ts +++ b/example/convex/_generated/api.d.ts @@ -8,6 +8,7 @@ * @module */ +import type * as gate from "../gate.js"; import type * as grants from "../grants.js"; import type * as orderQueries from "../orderQueries.js"; import type * as orderSummary from "../orderSummary.js"; @@ -22,6 +23,7 @@ import type { } from "convex/server"; declare const fullApi: ApiFromModules<{ + gate: typeof gate; grants: typeof grants; orderQueries: typeof orderQueries; orderSummary: typeof orderSummary; diff --git a/example/convex/gate.ts b/example/convex/gate.ts new file mode 100644 index 0000000..af1fbc9 --- /dev/null +++ b/example/convex/gate.ts @@ -0,0 +1,9 @@ +// The maintenance gate's four operator entries, registered by the gate library and re-exported here, +// so they are reached as internal.gate.closeGate, internal.gate.resumeGate, internal.gate.getGate and +// internal.gate.getGateAudit, with admin access only. +export { + closeGate, + getGate, + getGateAudit, + resumeGate, +} from "../../src/gate/index.js"; diff --git a/example/convex/schema.ts b/example/convex/schema.ts index 580f6ce..2bb81fd 100644 --- a/example/convex/schema.ts +++ b/example/convex/schema.ts @@ -1,10 +1,15 @@ import { defineSchema, defineTable } from "convex/server"; +import { auditTables } from "../../src/audit/index.js"; import { commandTables } from "../../src/command/index.js"; +import { gateTables } from "../../src/gate/index.js"; import { readModelTables, rowConventions } from "../../src/read-model/index.js"; import { orderSummaryFields } from "./orderSummary.js"; export default defineSchema({ - // The command library's receipts and grants. + // The command library's receipts, grants and tenant list. ...commandTables, + // The maintenance gate the pipeline's step 7 reads, and the audit records. See gate.ts. + ...gateTables, + ...auditTables, // The generation registry, and the rows of the order summary. See orderSummary.ts. ...readModelTables, orderSummaries: defineTable({ ...rowConventions, ...orderSummaryFields }) diff --git a/fixture/convex/_generated/api.d.ts b/fixture/convex/_generated/api.d.ts index 694907b..2e56366 100644 --- a/fixture/convex/_generated/api.d.ts +++ b/fixture/convex/_generated/api.d.ts @@ -14,6 +14,7 @@ import type * as depotRelay from "../depotRelay.js"; import type * as documentTitles from "../documentTitles.js"; import type * as failures from "../failures.js"; import type * as filing from "../filing.js"; +import type * as gate from "../gate.js"; import type * as grants from "../grants.js"; import type * as identity from "../identity.js"; import type * as limits from "../limits.js"; @@ -44,6 +45,7 @@ declare const fullApi: ApiFromModules<{ documentTitles: typeof documentTitles; failures: typeof failures; filing: typeof filing; + gate: typeof gate; grants: typeof grants; identity: typeof identity; limits: typeof limits; diff --git a/fixture/convex/gate.ts b/fixture/convex/gate.ts new file mode 100644 index 0000000..af1fbc9 --- /dev/null +++ b/fixture/convex/gate.ts @@ -0,0 +1,9 @@ +// The maintenance gate's four operator entries, registered by the gate library and re-exported here, +// so they are reached as internal.gate.closeGate, internal.gate.resumeGate, internal.gate.getGate and +// internal.gate.getGateAudit, with admin access only. +export { + closeGate, + getGate, + getGateAudit, + resumeGate, +} from "../../src/gate/index.js"; diff --git a/fixture/convex/schema.ts b/fixture/convex/schema.ts index eb5643c..c448058 100644 --- a/fixture/convex/schema.ts +++ b/fixture/convex/schema.ts @@ -1,6 +1,8 @@ import { defineSchema, defineTable } from "convex/server"; import { v } from "convex/values"; +import { auditTables } from "../../src/audit/index.js"; import { commandTables } from "../../src/command/index.js"; +import { gateTables } from "../../src/gate/index.js"; import { readModelTables, rowConventions } from "../../src/read-model/index.js"; import { documentTitleFields } from "./documentTitles.js"; import { documentSummaryFields } from "./summaries.js"; @@ -20,8 +22,11 @@ export default defineSchema({ "by_group", ["group"], ), - // The command library's receipts and grants. + // The command library's receipts, grants and tenant list. ...commandTables, + // The maintenance gate the pipeline's step 7 reads, and the audit records. See gate.ts. + ...gateTables, + ...auditTables, // The generation registry, and the rows of the fixture's document summary. See summaries.ts. ...readModelTables, documentSummaries: defineTable({ diff --git a/src/audit/index.ts b/src/audit/index.ts new file mode 100644 index 0000000..1651224 --- /dev/null +++ b/src/audit/index.ts @@ -0,0 +1,16 @@ +// The audit library: the audit record of a command and the operator audit record of a gate change. +export { + auditDecisionValidator, + auditKindValidator, + auditTables, + operatorAuditDoc, + operatorAuditFields, + operatorAuditKindValidator, +} from "./tables.js"; +export type { + AuditDataModel, + AuditRecord, + OperatorAuditRecord, +} from "./tables.js"; +export { writeAudit, writeOperatorAudit } from "./write.js"; +export type { AuditRecordInput, OperatorAuditInput } from "./write.js"; diff --git a/src/audit/tables.ts b/src/audit/tables.ts new file mode 100644 index 0000000..5424755 --- /dev/null +++ b/src/audit/tables.ts @@ -0,0 +1,77 @@ +// The audit tables of spec:operations.baseline-operations, as a fragment the app's schema.ts spreads +// into defineSchema: the audit record a command's step 10 writes, and the operator audit record of a +// gate change. auditRecords leads every index with the tenant; operatorAudit is operational data of +// the deployment, so its index leads with the scope. +import { + defineTable, + type DataModelFromSchemaDefinition, + type DocumentByName, + type SchemaDefinition, +} from "convex/server"; +import { v } from "convex/values"; +import { + actorValidator, + subjectRefValidator, +} from "../command/actor-and-scope.js"; +import { causedByValidator } from "../context/envelope.js"; +export const auditKindValidator = v.union( + v.literal("security"), + v.literal("business"), +); +export const auditDecisionValidator = v.union( + v.literal("applied"), + v.literal("businessFailure"), +); +export const operatorAuditKindValidator = v.union( + v.literal("gate.close"), + v.literal("gate.resume"), +); +export const operatorAuditFields = { + kind: operatorAuditKindValidator, + scopeKey: v.string(), + reason: v.string(), + generationId: v.optional(v.id("generations")), + operator: v.string(), + recordedAt: v.number(), +}; +export const auditTables = { + auditRecords: defineTable({ + tenantId: v.string(), + operationId: v.string(), + requestKey: v.optional(v.string()), + commandType: v.string(), + actor: actorValidator, + subject: subjectRefValidator, + kind: auditKindValidator, + decision: auditDecisionValidator, + causedBy: causedByValidator, + recordedAt: v.number(), + }) + .index("by_operation", ["tenantId", "operationId"]) + .index("by_subject", [ + "tenantId", + "subject.contextId", + "subject.streamType", + "subject.streamId", + "recordedAt", + ]) + .index("by_request_key", ["tenantId", "requestKey"]), + operatorAudit: defineTable(operatorAuditFields).index("by_scope", [ + "scopeKey", + "recordedAt", + ]), +}; +// The operatorAudit table's document validator with its system fields, the item of getGateAudit's page. +export const operatorAuditDoc = v.object({ + _id: v.id("operatorAudit"), + _creationTime: v.number(), + ...operatorAuditFields, +}); +export type AuditDataModel = DataModelFromSchemaDefinition< + SchemaDefinition +>; +export type AuditRecord = DocumentByName; +export type OperatorAuditRecord = DocumentByName< + AuditDataModel, + "operatorAudit" +>; diff --git a/src/audit/write.ts b/src/audit/write.ts new file mode 100644 index 0000000..77a0e49 --- /dev/null +++ b/src/audit/write.ts @@ -0,0 +1,40 @@ +// The two audit writers of spec:operations.baseline-operations. Each inserts one record inside the +// caller's mutation and lets every failure propagate, a validator refusal or a document too large +// among them, so the caller's mutation rolls back: audit fails closed. No catch surrounds either. +import type { GenericDatabaseWriter } from "convex/server"; +import type { GenericId } from "convex/values"; +import type { Actor, SubjectRef } from "../command/actor-and-scope.js"; +import type { CausedBy } from "../context/envelope.js"; +import type { AuditDataModel } from "./tables.js"; +// The auditRecords document without recordedAt, which writeAudit sets. +export type AuditRecordInput = { + tenantId: string; + operationId: string; + requestKey?: string; + commandType: string; + actor: Actor; + subject: SubjectRef; + kind: "security" | "business"; + decision: "applied" | "businessFailure"; + causedBy: CausedBy; +}; +export type OperatorAuditInput = { + kind: "gate.close" | "gate.resume"; + scopeKey: string; + reason: string; + generationId?: GenericId<"generations">; + operator: string; +}; +export async function writeAudit( + ctx: { db: GenericDatabaseWriter }, + record: AuditRecordInput, +): Promise> { + return ctx.db.insert("auditRecords", { ...record, recordedAt: Date.now() }); +} +// Called by the write pause's closeScope and resumeScope and by nothing else. +export async function writeOperatorAudit( + ctx: { db: GenericDatabaseWriter }, + record: OperatorAuditInput, +): Promise> { + return ctx.db.insert("operatorAudit", { ...record, recordedAt: Date.now() }); +} diff --git a/src/command/actor-and-scope.ts b/src/command/actor-and-scope.ts index dc50ee4..0d7c4d2 100644 --- a/src/command/actor-and-scope.ts +++ b/src/command/actor-and-scope.ts @@ -1,8 +1,10 @@ // The shapes of spec:command.actor-and-scope that the parent establishes and passes into a context: -// the actor, the tenant scope, the namespace and a worker's authority. Validators and types only, so -// the context library can import the actor's validator without carrying the parent's helpers; those, -// authorize, establishActor and the grant helpers, are in authority.ts. +// the actor, the tenant scope, the namespace and a worker's authority, and the operator an operator +// entry states. Validators, types and the operator's check only, so the context library can import the +// actor's validator without carrying the parent's helpers; those, authorize, establishActor and the +// grant helpers, are in authority.ts. import { v } from "convex/values"; +import { limitActorIdLength, utf8Length } from "../context/text.js"; export type ActorKind = "human" | "service" | "agent" | "reviewer" | "operator"; export type ActorRef = { kind: ActorKind; id: string }; // delegationRef names the obligation, approval or agent run the delegation rests on. @@ -70,3 +72,19 @@ export type AuthorizeInput = { permission: string; subject?: SubjectRef; }; +// The operator an operator entry that changes anything takes, and the type of every field that +// records it. A stated operator is what admin access chose to state and proves no identity. +export const operatorValidator = v.string(); +// The bound limitCallText gives an actor's id. +export const limitOperatorBytes = limitActorIdLength; +// An operator entry calls this first, so a refused entry has read and written nothing. +export function assertOperator(operator: string): string { + if (operator.trim() === "") + throw new Error("An operator entry needs a stated operator"); + const bytes = utf8Length(operator); + if (bytes > limitOperatorBytes) + throw new Error( + `The stated operator is ${bytes} bytes, above the limit of ${limitOperatorBytes}`, + ); + return operator; +} diff --git a/src/command/authority.ts b/src/command/authority.ts index 4d391a2..27df3f8 100644 --- a/src/command/authority.ts +++ b/src/command/authority.ts @@ -1,7 +1,8 @@ // The parent's authority helpers of spec:command.actor-and-scope and spec:command.tenancy-and-authority: // establishActor, the one reader of ctx.auth, authorize, the one reader of grants on a command path, -// authorizeQuery, which a parent query calls before it discloses anything, and the plain helpers an -// operator's function uses to write and delete a grant. None is registered here. +// authorizeQuery, which a parent query calls before it discloses anything, the plain helpers an +// operator's function uses to write and delete a grant, and nextTenant, which reads the tenant list. +// None is registered here. import type { GenericQueryCtx } from "convex/server"; import type { Actor, AuthorizeInput, SubjectRef } from "./actor-and-scope.js"; import { reject } from "./outcome-boundary.js"; @@ -84,11 +85,37 @@ export type GrantInput = { subject?: SubjectRef; grantedBy: string; }; +// Inserts the tenant's row in the tenant list when it has none, then the grant, in one mutation, so no +// tenant holds a grant without a row. export async function insertGrant( ctx: { db: MutationCtx["db"] }, grant: GrantInput, ): Promise { - return ctx.db.insert("grants", { ...grant, grantedAt: Date.now() }); + const now = Date.now(); + const tenant = await ctx.db + .query("tenants") + .withIndex("by_tenant", (q) => q.eq("tenantId", grant.tenantId)) + .first(); + if (tenant === null) + await ctx.db.insert("tenants", { + tenantId: grant.tenantId, + createdAt: now, + }); + return ctx.db.insert("grants", { ...grant, grantedAt: now }); +} +// The tenant after `after` in ascending tenant-ID order, or the first when `after` is null; null when +// none follows. One indexed read, so a pass over the whole deployment takes one tenant at a time. +export async function nextTenant( + ctx: { db: QueryCtx["db"] }, + after: string | null, +): Promise { + const row = await ctx.db + .query("tenants") + .withIndex("by_tenant", (q) => + after === null ? q : q.gt("tenantId", after), + ) + .first(); + return row === null ? null : row.tenantId; } // Revocation deletes the rows that grant this permission to this principal, on this subject when one // is named and otherwise on any; the next command's read finds none. Returns the number deleted. It diff --git a/src/command/declaration.ts b/src/command/declaration.ts index 39162e9..6b9f9b0 100644 --- a/src/command/declaration.ts +++ b/src/command/declaration.ts @@ -46,6 +46,8 @@ export type CommandDeclaration = { admission?: AdmissionPolicy; bounds?: Bounds; retention?: Retention; + // Asks step 10 for one audit record, which fails closed. Absent means none. + audit?: { kind: "security" | "business" }; }; export type PermissionPolicy = { permission: string; diff --git a/src/command/index.ts b/src/command/index.ts index 47e45f9..ee2b60b 100644 --- a/src/command/index.ts +++ b/src/command/index.ts @@ -2,8 +2,11 @@ export { actorKindValidator, actorValidator, + assertOperator, authorityValidator, callerNamespaceValidator, + limitOperatorBytes, + operatorValidator, subjectRefValidator, tenantScopeValidator, } from "./actor-and-scope.js"; @@ -24,6 +27,7 @@ export { establishActor, insertGrant, limitGrantsRead, + nextTenant, revokeGrant, } from "./authority.js"; export type { @@ -85,6 +89,7 @@ export type { MutationCtx, QueryCtx, Receipt, + Tenant, } from "./tables.js"; export { limitActorIdLength, diff --git a/src/command/pipeline.ts b/src/command/pipeline.ts index 5fb04cb..8741551 100644 --- a/src/command/pipeline.ts +++ b/src/command/pipeline.ts @@ -1,5 +1,9 @@ // The command pipeline of spec:command.command-pipeline: the steps between a caller's request and the // committed outcome, in one top-level mutation. Its only failure signal is a throw. +import type { + GenericDatabaseReader, + GenericDatabaseWriter, +} from "convex/server"; import { getConvexSize, v, type Validator, type Value } from "convex/values"; import { limitActorIdLength, @@ -24,6 +28,9 @@ import { type ReceiptKey, } from "./receipts.js"; import type { MutationCtx } from "./tables.js"; +import { writeAudit, type AuditDataModel } from "../audit/index.js"; +import { assertWritable, scopesOfUseCase } from "../gate/gate.js"; +import type { GateDataModel } from "../gate/tables.js"; import { fromSource, writeReadModels, @@ -178,8 +185,8 @@ function parse(decl: CommandDeclaration, call: PipelineCall) { if (refused !== null) reject({ ...refused, code: "invalidInput", commandType }); } -// Steps 1 and 3 to 11 for either entry; the public entry ran step 2 before it built the call. Step 7 -// reads no gate, and steps 10 and 11 write no audit record and emit no diagnostic. +// Steps 1 and 3 to 11 for either entry; the public entry ran step 2 before it built the call. Step 11 +// emits no diagnostic. export async function runPipeline( ctx: MutationCtx, decl: CommandDeclaration, @@ -265,7 +272,13 @@ export async function runPipeline( : { retryAfterMs: admission.retryAfterMs }), }); } - // Step 7: the operation, minted once, before the first context call. + // Step 7: the gate, read once, and only then the operation, minted once, before the first context + // call. The library types ctx with the command tables; every composition that registers a command + // holds the gate's table too, and at run time ctx is that composition's own. + await assertWritable( + ctx as unknown as { db: GenericDatabaseReader }, + scopesOfUseCase(tenantId, decl.writes), + ); const operation: OperationRef = { operationId: crypto.randomUUID(), causedBy: call.causedBy ?? { kind: "command", commandType }, @@ -318,6 +331,39 @@ export async function runPipeline( }, decl.retention, ); + // Step 10, the audit record of a declaration that sets audit, every field from the command alone. + if (decl.audit !== undefined) { + const first = executed.versions[0]; + const subject = + decl.permission.subjectFrom?.(input) ?? + (first === undefined + ? undefined + : { + contextId: first.contextId, + streamType: first.streamType, + streamId: first.streamId, + }); + if (subject === undefined) + throw new Error( + `${commandType} sets audit, declares no subjectFrom and returned no version, so its audit record has no subject`, + ); + await writeAudit( + ctx as unknown as { db: GenericDatabaseWriter }, + { + tenantId, + operationId: operation.operationId, + ...(call.requestKey === undefined + ? {} + : { requestKey: call.requestKey }), + commandType, + actor, + subject, + kind: decl.audit.kind, + decision: executed.kind, + causedBy: operation.causedBy, + }, + ); + } // Step 11. return { kind: executed.kind, diff --git a/src/command/tables.ts b/src/command/tables.ts index 579cd7b..4944462 100644 --- a/src/command/tables.ts +++ b/src/command/tables.ts @@ -1,5 +1,6 @@ -// The parent's tables of spec:command.receipt-table and spec:command.actor-and-scope, as a fragment the -// app's schema.ts spreads into defineSchema. Every index leads with tenantId. +// The parent's tables of spec:command.receipt-table and spec:command.actor-and-scope, receipts, grants +// and the tenant list, as a fragment the app's schema.ts spreads into defineSchema. Every index leads +// with tenantId. import { defineTable, type DataModelFromSchemaDefinition, @@ -50,6 +51,12 @@ export const commandTables = { }) .index("by_principal", ["tenantId", "principalKind", "principalId"]) .index("by_permission", ["tenantId", "permission"]), + // The tenant list: one row per tenant, inserted by insertGrant with the tenant's first grant and + // never deleted. + tenants: defineTable({ + tenantId: v.string(), + createdAt: v.number(), + }).index("by_tenant", ["tenantId"]), }; // The data model the library reads and writes. The app's own data model holds these tables. export type CommandDataModel = DataModelFromSchemaDefinition< @@ -60,3 +67,4 @@ export type QueryCtx = GenericQueryCtx; export type Receipt = DocumentByName; export type Grant = DocumentByName; export type GrantId = GenericId<"grants">; +export type Tenant = DocumentByName; diff --git a/src/gate/entries.ts b/src/gate/entries.ts new file mode 100644 index 0000000..d1560e7 --- /dev/null +++ b/src/gate/entries.ts @@ -0,0 +1,105 @@ +// The four operator entries of spec:application.write-pause, registered here and re-exported by a +// composition's gate module, so they are reached as internal.gate.closeGate and so on. Each follows the +// operator entry contract of spec:command.actor-and-scope: an internal function, so only admin access +// reaches it, that reads no grant and no ctx.auth; the two that change the gate check the stated +// operator first and record it. An operator entry closes and resumes only "all" or a tenant scope; a +// source scope is closed and reopened by the rebuild alone. Every refusal is a plain Error. +import { + internalMutationGeneric, + internalQueryGeneric, + paginationOptsValidator, + paginationResultValidator, + type MutationBuilder, + type QueryBuilder, +} from "convex/server"; +import { v } from "convex/values"; +import { operatorAuditDoc } from "../audit/tables.js"; +import { + assertOperator, + operatorValidator, +} from "../command/actor-and-scope.js"; +import { limitIdLength, utf8Length } from "../context/text.js"; +import { + closeScope, + limitGateReasonBytes, + resumeScope, + restoreDoorClosed, +} from "./gate.js"; +import { closedEntryValidator, type GateChangeDataModel } from "./tables.js"; +// The builders a composition's generated server exports are these, typed with its data model. The +// library types them with the tables it owns, which the composition's schema holds. +const internalMutation: MutationBuilder = + internalMutationGeneric; +const internalQuery: QueryBuilder = + internalQueryGeneric; +// At most 100 audit records a page. +export const limitOperatorQuery = 100; +// "all", or "tenant:" followed by a tenant ID of 1 to limitIdLength bytes. The tenant list is not read, +// so a tenant may be closed before its first grant. +function assertOperatorScope(entry: string, scopeKey: string) { + const tenant = scopeKey.startsWith("tenant:") + ? utf8Length(scopeKey.slice("tenant:".length)) + : 0; + if (scopeKey !== "all" && (tenant < 1 || tenant > limitIdLength)) + throw new Error( + `${entry} takes the scope all or a tenant scope, and ${scopeKey} is neither`, + ); +} +export const closeGate = internalMutation({ + args: { + scopeKey: v.string(), + reason: v.string(), + operator: operatorValidator, + }, + returns: v.null(), + handler: async (ctx, { scopeKey, reason, operator }) => { + assertOperator(operator); + assertOperatorScope("closeGate", scopeKey); + const bytes = utf8Length(reason); + if (bytes < 1 || bytes > limitGateReasonBytes) + throw new Error( + `The reason must be between 1 and ${limitGateReasonBytes} bytes`, + ); + await closeScope(ctx, scopeKey, reason, operator); + return null; + }, +}); +export const resumeGate = internalMutation({ + args: { scopeKey: v.string(), operator: operatorValidator }, + returns: v.null(), + handler: async (ctx, { scopeKey, operator }) => { + assertOperator(operator); + assertOperatorScope("resumeGate", scopeKey); + await resumeScope(ctx, scopeKey, operator); + return null; + }, +}); +// Reads only, so it takes no operator. +export const getGate = internalQuery({ + args: {}, + returns: v.object({ + restore: v.boolean(), + closed: v.array(closedEntryValidator), + }), + handler: async (ctx) => { + const gate = await ctx.db + .query("maintenanceGates") + .withIndex("by_key", (q) => q.eq("key", "gates")) + .unique(); + return { restore: restoreDoorClosed(), closed: gate?.closed ?? [] }; + }, +}); +// Who closed and who reopened one scope, newest first; a page asking for more than 100 gets 100. +export const getGateAudit = internalQuery({ + args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, + returns: paginationResultValidator(operatorAuditDoc), + handler: async (ctx, { scopeKey, paginationOpts }) => + ctx.db + .query("operatorAudit") + .withIndex("by_scope", (q) => q.eq("scopeKey", scopeKey)) + .order("desc") + .paginate({ + ...paginationOpts, + numItems: Math.min(paginationOpts.numItems, limitOperatorQuery), + }), +}); diff --git a/src/gate/gate.ts b/src/gate/gate.ts new file mode 100644 index 0000000..4eb1120 --- /dev/null +++ b/src/gate/gate.ts @@ -0,0 +1,160 @@ +// The writer check and the two gate writers of spec:application.write-pause. gateAllows is the one gate +// read: the restore door through the environment first, then the gate document. assertWritable is +// gateAllows plus the throw, called by the command pipeline's step 7 and by nothing else. closeScope and +// resumeScope are the only writers of the gate document, and each writes its operator audit record in +// the caller's mutation with no catch around it, so a failed insert changes nothing. +import type { + GenericDatabaseReader, + GenericDatabaseWriter, +} from "convex/server"; +import type { GenericId } from "convex/values"; +import { writeOperatorAudit } from "../audit/write.js"; +import { refuseTransient } from "../command/outcome-boundary.js"; +import type { SourceRef } from "../read-model/write.js"; +import type { + ClosedEntry, + GateChangeDataModel, + GateDataModel, + GateDocument, +} from "./tables.js"; +// "all", "tenant:" + tenantId, or "source:" + contextId + ":" + streamType. +export type ScopeKey = "all" | (string & {}); +export type GateAnswer = + { allowed: true } | { allowed: false; scopeKey: ScopeKey; reason: string }; +// At most 16 scopes closed at once, so the document stays a few kilobytes. +export const limitClosedScopes = 16; +// A reason is 1 to 256 bytes of UTF-8, so the refusal's message stays under a kilobyte. +export const limitGateReasonBytes = 256; +export const tenantScope = (tenantId: string): ScopeKey => `tenant:${tenantId}`; +export const sourceScope = (source: SourceRef): ScopeKey => + `source:${source.contextId}:${source.streamType}`; +// "all", the tenant scope and one source scope per stream type the declaration writes, in that order. +export function scopesOfUseCase( + tenantId: string, + writes: readonly SourceRef[], +): ScopeKey[] { + return ["all", tenantScope(tenantId), ...writes.map(sourceScope)]; +} +// The one point read of the gate document; an absent document is no closed scope. +async function readGate(db: GenericDatabaseReader) { + return db + .query("maintenanceGates") + .withIndex("by_key", (q) => q.eq("key", "gates")) + .unique(); +} +// Only the exact value "restore" closes the door. No component reads the variable. +export function restoreDoorClosed(): boolean { + return process.env.MAINTENANCE_MODE === "restore"; +} +// An entry is exempt only when the read states a generation, the entry names one, and they are equal. +const exempt = ( + entry: ClosedEntry, + allowGeneration?: GenericId<"generations">, +) => + allowGeneration !== undefined && + entry.generationId !== undefined && + entry.generationId === allowGeneration; +export async function gateAllows( + ctx: { db: GenericDatabaseReader }, + scopes: readonly ScopeKey[], + allowGeneration?: GenericId<"generations">, +): Promise { + if (restoreDoorClosed()) + return { allowed: false, scopeKey: "all", reason: "restore" }; + const gate = await readGate(ctx.db); + const entry = gate?.closed.find( + (closed) => + scopes.includes(closed.scopeKey) && !exempt(closed, allowGeneration), + ); + return entry === undefined + ? { allowed: true } + : { allowed: false, scopeKey: entry.scopeKey, reason: entry.reason }; +} +export async function assertWritable( + ctx: { db: GenericDatabaseReader }, + scopes: readonly ScopeKey[], +): Promise { + const answer = await gateAllows(ctx, scopes); + if (!answer.allowed) + refuseTransient({ + code: "writePaused", + message: `write paused for ${answer.scopeKey}: ${answer.reason}`, + }); +} +type GateWriter = { db: GenericDatabaseWriter }; +async function saveClosed( + ctx: GateWriter, + gate: GateDocument | null, + closed: ClosedEntry[], + now: number, +) { + if (gate === null) + await ctx.db.insert("maintenanceGates", { + key: "gates", + closed, + updatedAt: now, + }); + else await ctx.db.patch(gate._id, { closed, updatedAt: now }); +} +// Called by closeGate and by the rebuild's startGeneration, which passes the generation row it inserted. +export async function closeScope( + ctx: GateWriter, + scopeKey: ScopeKey, + reason: string, + operator: string, + generationId?: GenericId<"generations">, +): Promise { + const gate = await readGate(ctx.db); + const closed = gate?.closed ?? []; + if (closed.some((entry) => entry.scopeKey === scopeKey)) + throw new Error(`The scope ${scopeKey} is already closed`); + if (closed.length >= limitClosedScopes) + throw new Error( + `The gate holds ${limitClosedScopes} closed scopes, its limit`, + ); + const now = Date.now(); + const generation = generationId === undefined ? {} : { generationId }; + await saveClosed( + ctx, + gate, + [ + ...closed, + { scopeKey, reason, ...generation, changedAt: now, changedBy: operator }, + ], + now, + ); + await writeOperatorAudit(ctx, { + kind: "gate.close", + scopeKey, + reason, + ...generation, + operator, + }); +} +// Called by resumeGate and by the rebuild's abortGeneration and switchGeneration. The audit record +// carries the reason and the generation of the entry it removed. +export async function resumeScope( + ctx: GateWriter, + scopeKey: ScopeKey, + operator: string, +): Promise { + const gate = await readGate(ctx.db); + const entry = gate?.closed.find((closed) => closed.scopeKey === scopeKey); + if (gate === null || entry === undefined) + throw new Error(`The scope ${scopeKey} is not closed`); + await saveClosed( + ctx, + gate, + gate.closed.filter((closed) => closed !== entry), + Date.now(), + ); + await writeOperatorAudit(ctx, { + kind: "gate.resume", + scopeKey, + reason: entry.reason, + ...(entry.generationId === undefined + ? {} + : { generationId: entry.generationId }), + operator, + }); +} diff --git a/src/gate/index.ts b/src/gate/index.ts new file mode 100644 index 0000000..896bddf --- /dev/null +++ b/src/gate/index.ts @@ -0,0 +1,29 @@ +// The maintenance gate: its table, the writer check every command and background batch makes, the two +// gate writers and the four operator entries a composition re-exports from its gate module. +export { + closeGate, + getGate, + getGateAudit, + limitOperatorQuery, + resumeGate, +} from "./entries.js"; +export { + assertWritable, + closeScope, + gateAllows, + limitClosedScopes, + limitGateReasonBytes, + restoreDoorClosed, + resumeScope, + scopesOfUseCase, + sourceScope, + tenantScope, +} from "./gate.js"; +export type { GateAnswer, ScopeKey } from "./gate.js"; +export { closedEntryValidator, gateTables } from "./tables.js"; +export type { + ClosedEntry, + GateChangeDataModel, + GateDataModel, + GateDocument, +} from "./tables.js"; diff --git a/src/gate/tables.ts b/src/gate/tables.ts new file mode 100644 index 0000000..ad0651e --- /dev/null +++ b/src/gate/tables.ts @@ -0,0 +1,35 @@ +// The maintenance gate's table of spec:application.write-pause, as a fragment every composition that +// registers a command spreads into defineSchema, because the pipeline's step 7 reads it. It holds at +// most one document, whose closed entries are in the order they were closed. +import { + defineTable, + type DataModelFromSchemaDefinition, + type DocumentByName, + type SchemaDefinition, +} from "convex/server"; +import { v } from "convex/values"; +import type { auditTables } from "../audit/tables.js"; +// changedBy is the stated operator and changedAt the closing mutation's Date.now(). +export const closedEntryValidator = v.object({ + scopeKey: v.string(), + reason: v.string(), + generationId: v.optional(v.id("generations")), + changedAt: v.number(), + changedBy: v.string(), +}); +export const gateTables = { + maintenanceGates: defineTable({ + key: v.literal("gates"), + closed: v.array(closedEntryValidator), + updatedAt: v.number(), + }).index("by_key", ["key"]), +}; +export type GateDataModel = DataModelFromSchemaDefinition< + SchemaDefinition +>; +export type GateDocument = DocumentByName; +export type ClosedEntry = GateDocument["closed"][number]; +// What a gate change writes: the gate document and its operator audit record, in one mutation. +export type GateChangeDataModel = DataModelFromSchemaDefinition< + SchemaDefinition +>; diff --git a/tests/pure/command.test.ts b/tests/pure/command.test.ts index 8849f2a..988f953 100644 --- a/tests/pure/command.test.ts +++ b/tests/pure/command.test.ts @@ -372,7 +372,7 @@ test("pure: classifyThrown tells a rejection from a transient refusal from a tec expect(classifyThrown(error)).toEqual({ kind: "technical", error }); }); -test("pure: the receipts and grants indexes lead with tenantId, and a receipt has no field for input or result", () => { +test("pure: the receipts, grants and tenants indexes lead with tenantId, and a receipt has no field for input or result", () => { const indexes = Object.values(commandTables).flatMap((table) => table[" indexes"](), ); @@ -384,6 +384,7 @@ test("pure: the receipts and grants indexes lead with tenantId, and a receipt ha ["by_tenant_expiry", ["tenantId", "expiresAt"]], ["by_principal", ["tenantId", "principalKind", "principalId"]], ["by_permission", ["tenantId", "permission"]], + ["by_tenant", ["tenantId"]], ]); expect(Object.keys(commandTables.receipts.validator.fields).sort()).toEqual([ "actorId", From 8df8f4db9e0108dc7718eb11b716e5c7f8455418 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:48:53 +0200 Subject: [PATCH 034/155] design: the contention run's dataset layout and its rule on caller retries in the contention shape Co-Authored-By: Claude Opus 5.5 --- design/specs/application/first-experiment.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 75f0b18..b44d365 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -113,7 +113,7 @@ The experiment is an application, a test suite and a report. The application is - captureRowsLeftBehind: rows left behind are the documents a run creates that are not state, events, receipts or read-model rows; record their before, after and created counts by table, separately from setup, including markers, registry, gate and audit rows, and say `not deployed` for a table absent from the deployed schema (E-47, First experiment, D19) - captureRetries: one completion with `willRetry` true and an `occInfo` is one execution refused at commit; its rerun has another completion with the same backend request ID; record these executions, the tables named by `occInfo` with the component that holds each, the engine's own `retryCount` of each refused execution, which numbers them from zero, each command's terminal record and final outcomes separately, and count terminal optimistic-concurrency failures as commands the engine failed; the variable number of reruns is an observation, never the difference between two cells (E-47, F1) - datasetSizes: 1 line, 10 lines and 100 lines, the maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) -- contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with stock for one, so that one is applied and the rest are rejected for short stock or failed by the engine after its bounded retries, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry (First experiment, Sc L1-11, F1, E-5, E-47) +- contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with stock for one, so that one is applied and the rest are rejected for short stock or failed by the engine after its bounded retries, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry; the shared stock item is the first line of every order and each other line is on a stock item of that command alone, received once with quantity two, and no caller retries during the cell, while the caller may repeat a command the engine failed under its same request key (First experiment, Sc L1-11, F1, E-5, E-47) - limitOrderLines: 100 lines per `PlaceOrder`, the largest order the example supports and the bound its declaration carries, chosen under the 1 second timeout, the F13 ceilings and the adapter's stream and byte bounds; it is the example's promise and no platform limit, and the experiment reports documents read and written per line (OQ3, F13) - limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate read, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer (E-47, F13) From c4b15430181ddcde3ceb5cf863cd0064a560a0b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:48:58 +0200 Subject: [PATCH 035/155] design: first-run sentences of the Probe 7 examples that held, the probe plan's run sentence and the temporary deploy's composition claim Co-Authored-By: Claude Opus 5.5 --- ...lude-pending-scheduled-functions.probe-7-fresh-import.sdp.md | 2 +- ...e-pending-scheduled-functions.probe-7-in-place-import.sdp.md | 2 +- .../facts/f13-transactions-have-limits.probe-7-arguments.sdp.md | 2 +- ...uled-functions-table-shows-failed-runs.probe-7-states.sdp.md | 2 +- design/specs/facts/probe-plan.sdp.md | 2 +- design/specs/platform/native-harness.sdp.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md index 0e42950..90af63c 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md @@ -26,5 +26,5 @@ And each stored id validates as an argument and on a later write {valid: true}, ## Verification — executable -- The value is observed, not expected: the expectation written before the first run was restored documents and empty scheduler tables in a fresh destination, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed that result in the parent, Orders and Inventory, with stored scheduler IDs still validating. +- On the first run of this example, on 2026-10-02 on release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, the bound values held: the documents came back in the parent, Orders and Inventory, the scheduler tables stayed empty, and the stored scheduler ids still validated. - The exported snapshot is the archive. Full document equality includes IDs and creation times. Hosted dashboard restore, file storage, Workpool and Workflow are not run. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md index 1e2052e..e8b1be4 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md @@ -27,6 +27,6 @@ And each restored scheduler reference resolves to the kept row {found: true} and ## Verification — executable -- The value is observed, not expected: the expectation written before the first run was replacement of documents with destination schedules kept, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed unchanged scheduler rows in all scopes and kept pending reactions reading exported data at or after their due time. +- On the first run of this example, on 2026-10-02 on release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, the bound values held: the scheduler rows were unchanged in all three scopes and the kept pending reactions read the restored value at or after their time. - The test changes business data in the parent and both contexts before import and fails if it does not reach that boundary. It records execution times; it binds only that execution is no earlier than the due time. - The exported snapshot is the archive. Hosted dashboard restore, file storage, Workpool and Workflow are not run. diff --git a/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md b/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md index 6a73304..d5d566d 100644 --- a/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.probe-7-arguments.sdp.md @@ -33,6 +33,6 @@ And the shorter warning accepts accounted sizes {shortSample: 4194254} and {shor - The value is observed, not expected: the expectation written before the first run was a hard 4 MiB per-call limit, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed a 16777216-byte sum enforced at runAfter while larger single calls warn and return. - The tested objects account for UTF-8 content plus 14 bytes per call and two bytes per array element. Newline and quote characters distinguish this accounting from JSON escapes. -- The value is observed, not expected: the expectation written before the first run of the warning boundary was no per-call warning at 3355443 accounted bytes, the shorter warning from 3355444 through 4194304 and the future hard error form from 4194305, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed those boundaries with the calls still accepted. +- The warning bounds are observed and were not expected: the first run on release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0 on 2026-10-02 found the per-call warning in the completion record's log lines, and a later run of this example searched for its onset and bound it; the shorter warning starts at 80 percent of 4 MiB, the future hard error form one byte above 4 MiB, and every call stays accepted. - The warning names a future hard error at 4 MiB, which remains the design ceiling per call. - The test must reach accepted and refused neighbours. Hosted limits are not run. diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md index ffb1084..262d4ae 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-states.sdp.md @@ -26,5 +26,5 @@ And all scheduler rows remain unchanged across the interval {retained: true} ## Verification — executable -- The value is observed, not expected: the expectation written before the first run was all five states with completion times only on terminal rows, and the run on release `precompiled-2026-09-28-5c7cb5b` with Convex 1.46.0 on 2026-10-02 showed those states and fields in all three scopes. +- On the first run of this example, on 2026-10-02 on release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, the bound values held in all three scopes. - The held action supplies inProgress. The test records the retention interval; seven-day retention and expiry and hosted retention are not run. diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index 313cf07..4efd956 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -20,7 +20,7 @@ relations: Feature · Detail: full transcription · Traces: Probe 1 to Probe 7, F4, F5, F12, F13, F14, F15, F16, F17, S13, E-16. -Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`; Probe 7 observes the states and a short retention interval, scheduled-argument accounting, failed-function scan cost and local CLI replacement, while seven-day expiry and hosted dashboard restore remain open. +Seven probes, in the order the decisions need them, each one small test on a native backend. A probe turns an assumed fact into a probed one, or measures a size the docs leave open. The recheck of 2026-09-30 narrowed two of them and left the rest standing: the limits page states no limit on nested calls, so Probe 4 stands; the application-errors page does not state that `ConvexError` data survives a component boundary, so Probe 2 stands; the scheduling page documents the states and the 7-day retention of `_scheduled_functions`; Probe 7 confirms the states over a short interval and answers what a local CLI replacement leaves, while seven-day expiry and hosted dashboard restore remain open. Slice S0 ran probes 1 to 5 on a native backend on 2026-10-01, Probe 7 ran on 2026-10-02, and each result is recorded on its fact; Probe 6 has not run. The probes are not the first experiment. The experiment builds Layers 0 to 2 and measures cost; the probes settle single facts and can run before it, in a fixture app that is separate from the example app. diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 268d572..17dd8e6 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -78,7 +78,7 @@ The harness is plain test support with no Convex function in it. The entries who - fnFixtureBackend: [extension] `fixtureBackend(options?: { issuer?: string; executable?: Executable; parentDirectory?: string; deploy?: boolean }): Promise` starts a backend, deploys the fixture composition unless `deploy` is `false`, and removes the backend when the test finishes, however it finishes; the executable and parent directory options let the harness's own tests exercise the same cleanup path (E-15, E-13) - fnProductionBackend: [extension] `productionBackend(options?: { issuer?: string; deploy?: boolean }): Promise` starts a backend, deploys the production composition unless `deploy` is `false`, and removes the backend when the test finishes, however it finishes (E-15, E-13) - fnMeasure: [extension] `measure(name: string, value: JsonValue): void` stores a measurement on the running test (E-15) -- typeBackendFacts: [extension] `interface BackendFacts { composition: "fixture" | "production" | null; installedLayers: string[]; executable: { release: string | null; sha256: string; source: "cache" | "CONVEX_BACKEND_BINARY" }; identitySource: { kind: "fixture issuer"; issuer: string }; environment: string[]; dataset: "empty at start" }` uses null after a temporary deployment; its run measurement names the base composition, added functions and schema hashes instead of claiming the plain composition was deployed (E-15, E-13) +- typeBackendFacts: [extension] `interface BackendFacts { composition: "fixture" | "production" | null; installedLayers: string[]; executable: { release: string | null; sha256: string; source: "cache" | "CONVEX_BACKEND_BINARY" }; identitySource: { kind: "fixture issuer"; issuer: string }; environment: string[]; dataset: "empty at start" }` says `null` for the composition after `deployTemporary`, so a record never claims the plain composition for a backend running a copy; the test that deployed the copy records what it added as a measurement (E-15, E-13) - typeNativeTestEntry: [extension] `interface NativeTestEntry { project: "types" | "pure" | "simulator" | "native"; name: string; file: string; result: "passed" | "failed" | "skipped"; durationMs: number | null; errors: string[]; backends: BackendFacts[]; measurements: { name: string; value: JsonValue }[] }`; a run that includes the native project records every test it ran, of every project, each under the project that ran it (E-15) - typeNativeRunRecord: [extension] `interface NativeRunRecord { tier: "native"; commit: string; clean: boolean; command: string; startedAt: string; finishedAt: string; result: "passed" | "failed" | "interrupted"; versions: Record; tests: NativeTestEntry[]; unhandledErrors: string[] }`; it holds no field for adjusted configuration, and a run that adjusts configuration is the trigger that adds one (E-15, Acceptance scenarios) - compositionLayout: [extension] a composition is a name, a project directory and a functions directory; the Convex CLI reads `convex.json` and `package.json` from its working directory and takes no functions directory any other way, so the harness runs every CLI call in the composition's project directory; the fixture composition's project directory is the repository root, whose `convex.json` names `fixture/convex/`, and the production composition's is `example/`, whose `convex.json` names `convex/` and whose `package.json` is a symbolic link to the root's, so the repository has one `package.json` (E-15) From aa7405122a6b731371a89b59c4657f29ff911fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:50:00 +0200 Subject: [PATCH 036/155] build: the first-rebuild example bound to its steps and skipped until the rebuild is built Co-Authored-By: Claude Opus 5.5 --- ...tivation-makes-read-model-writable.test.ts | 93 +++++-------------- 1 file changed, 21 insertions(+), 72 deletions(-) diff --git a/tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts b/tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts index 0081d53..c64c091 100644 --- a/tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts +++ b/tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts @@ -4,20 +4,8 @@ import { testAnchorId, } from "@libar-dev/software-delivery-protocol"; import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; -import { expect } from "vitest"; +import { describe } from "vitest"; import { firstActivationMakesReadModelWritableContract as contract } from "../../generated/contracts/application.generation-registry.first-activation-makes-read-model-writable.contract.js"; -import { api } from "../../example/convex/_generated/api.js"; -import { orderSummary } from "../../example/convex/orderSummary.js"; -import { required } from "../../harness/native.js"; -import { - activateOrderSummary, - caught, - line, - orderWorld, - receiveStock, - tenantId, - type OrderWorld, -} from "./order-use-case.js"; const anchor = specTest({ id: testAnchorId( "test:application.generation-registry.first-activation-makes-read-model-writable", @@ -27,63 +15,24 @@ const anchor = specTest({ ), }); void anchor; -// The production composition's order summary starts with no generation row. The first activation, -// run by an operator with admin access, makes generation 1 the active one; a second activation is -// refused; and PlaceOrder then writes its summary row into generation 1. -interface World { - order?: OrderWorld; - activated?: unknown; -} -const generations = (world: World) => - required(world.order, "the backend").backend.admin.readTable("generations"); -bindExample(contract, (): World => ({}), { - "a read model with {generationRows} generation rows": async ( - world, - { generationRows }, - ) => { - world.order = await orderWorld({ activate: false }); - expect(await generations(world)).toHaveLength(generationRows); - }, - "an operator runs the first activation with admin access": async (world) => { - const { backend } = required(world.order, "the backend"); - world.activated = await activateOrderSummary(backend); - }, - "generation {generation} of the read model is {state}": async ( - world, - { generation, state }, - ) => { - expect(world.activated).toBe(generation); - expect(await generations(world)).toMatchObject([ - { - readModel: orderSummary.name, - generation, - projectionVersion: orderSummary.projection.version, - state, - startedBy: { kind: "operator", id: "native-test" }, - }, - ]); - }, - // The refusal names the read model, its generation and its state, and the registry is unchanged. - // The read model is writable: PlaceOrder writes the order's summary row into generation 1. - "a second first activation is {second}": async (world, { second }) => { - expect(second).toBe("refused"); - const order = required(world.order, "the backend"); - const registry = await generations(world); - const error = await caught(activateOrderSummary(order.backend)); - expect(String(error)).toContain( - "Read model orderSummary already has generation 1, which is active", - ); - expect(await generations(world)).toEqual(registry); - await receiveStock(order, [{ stockItemId: "sku-1", quantity: 5 }]); - expect( - await order.client.mutation(api.ordering.placeOrder, { - tenantId, - requestKey: "k-1", - input: { orderId: "order-1", lines: [line("sku-1", 2)] }, - }), - ).toMatchObject({ kind: "applied", replayed: false }); - expect(await order.backend.admin.readTable("orderSummaries")).toMatchObject( - [{ tenantId, generation: 1, key: "order-1", status: "placed" }], - ); - }, +// A read model is installed through its first rebuild: startGeneration, its batches and +// switchGeneration. The rebuild is not built, so the example is bound to its steps and skipped, and +// each step throws until the rebuild's build fills it in. +const notBuilt = (step: string): never => { + throw new Error(`Not built: ${step}`); +}; +type World = Record; +describe.skip("the first rebuild installs a read model", () => { + bindExample(contract, (): World => ({}), { + "a read model with {generationRows} generation rows and {orders} subjects that already have history": + () => notBuilt("a read model with history and no generation row"), + "an operator starts a generation, its batches finish and the operator switches it": + () => notBuilt("startGeneration, its batches and switchGeneration"), + "generation {generation} of the read model is {state}": () => + notBuilt("the generation row after the switch"), + "the read model holds {rows} rows for the subjects that existed before it": + () => notBuilt("the rows of the subjects that existed before"), + "a second start while generation 1 is being built is {second}": () => + notBuilt("the refusal of a second start"), + }); }); From 8097cc84ed78cb3c39caac0fee589a52c956258b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:50:15 +0200 Subject: [PATCH 037/155] test: the gate, its audit records, the operator entries and the tenant list at each tier Co-Authored-By: Claude Opus 5.5 --- tests/native/gate.test.ts | 227 ++++++++++++ tests/pure/gate.test.ts | 76 ++++ tests/simulator/gate.test.ts | 683 +++++++++++++++++++++++++++++++++++ tests/types/gate.test-d.ts | 51 +++ 4 files changed, 1037 insertions(+) create mode 100644 tests/native/gate.test.ts create mode 100644 tests/pure/gate.test.ts create mode 100644 tests/simulator/gate.test.ts create mode 100644 tests/types/gate.test-d.ts diff --git a/tests/native/gate.test.ts b/tests/native/gate.test.ts new file mode 100644 index 0000000..37161e5 --- /dev/null +++ b/tests/native/gate.test.ts @@ -0,0 +1,227 @@ +// The maintenance gate on the native backend, over the fixture composition: a closed tenant refuses a +// public command and leaves another tenant writable, the restore door set through the deployment's +// environment refuses the next write with no deploy, a close races commands in flight, and every +// operator action has its audit record (spec:application.write-pause, spec:operations.baseline-operations). +import { getFunctionName } from "convex/server"; +import { ConvexError, type Value } from "convex/values"; +import { expect, test } from "vitest"; +import { api, internal } from "../../fixture/convex/_generated/api.js"; +import { permissions } from "../../fixture/convex/depotCommands.js"; +import type { Backend } from "../../harness/backend.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend } from "../../harness/native.js"; +async function userIn(backend: Backend, tenantId: string, subject: string) { + await backend.admin.run(getFunctionName(internal.grants.grant), { + tenantId, + principalKind: "human", + principalId: `${backend.issuer.issuer}|${subject}`, + permission: permissions.documents, + grantedBy: "native-test", + }); + return ordinaryClient(backend.url, { + token: await backend.issuer.token(subject), + }); +} +type Client = Awaited>; +const create = ( + client: Client, + tenantId: string, + documentId: string, + requestKey: string, +) => + client.mutation(api.depotCommands.createDocument, { + tenantId, + requestKey, + input: { documentId, title: "Report" }, + }); +async function thrownData(promise: Promise): Promise { + const error = await promise.then( + () => { + throw new Error("The call did not fail"); + }, + (thrown: unknown) => thrown, + ); + expect(error).toBeInstanceOf(ConvexError); + return (error as ConvexError).data; +} +const paused = (message: string) => ({ + kind: "transient", + code: "writePaused", + message, +}); +const gateAudit = async (backend: Backend, scopeKey: string) => + ( + (await backend.admin.run(getFunctionName(internal.gate.getGateAudit), { + scopeKey, + paginationOpts: { numItems: 100, cursor: null }, + })) as { page: Record[] } + ).page; + +test("native: a closed tenant refuses a public command with writePaused and stores nothing, another tenant writes, and the same request key applies after the resume; each change has its audit record", async () => { + const backend = await fixtureBackend(); + const { admin } = backend; + const first = await userIn(backend, "t-1", "user-1"); + const second = await userIn(backend, "t-2", "user-2"); + await admin.run(getFunctionName(internal.gate.closeGate), { + scopeKey: "tenant:t-1", + reason: "move", + operator: "ops-close", + }); + expect(await thrownData(create(first, "t-1", "doc-1", "k-1"))).toEqual( + paused("write paused for tenant:t-1: move"), + ); + expect(await admin.readTable("receipts")).toEqual([]); + expect(await admin.readTable("events", { component: "depot" })).toEqual([]); + expect(await create(second, "t-2", "doc-1", "k-1")).toMatchObject({ + kind: "applied", + }); + await admin.run(getFunctionName(internal.gate.resumeGate), { + scopeKey: "tenant:t-1", + operator: "ops-resume", + }); + expect(await create(first, "t-1", "doc-1", "k-1")).toMatchObject({ + kind: "applied", + replayed: false, + versions: [expect.objectContaining({ streamId: "doc-1", version: 1 })], + }); + expect(await gateAudit(backend, "tenant:t-1")).toMatchObject([ + { + kind: "gate.resume", + scopeKey: "tenant:t-1", + reason: "move", + operator: "ops-resume", + }, + { + kind: "gate.close", + scopeKey: "tenant:t-1", + reason: "move", + operator: "ops-close", + }, + ]); + expect(await gateAudit(backend, "all")).toEqual([]); +}); + +test("native: MAINTENANCE_MODE set to restore on the running backend refuses the next write of every tenant and entry with no deploy, and any other value lets the same request key apply", async () => { + const backend = await fixtureBackend(); + const { admin } = backend; + const first = await userIn(backend, "t-1", "user-1"); + const second = await userIn(backend, "t-2", "user-2"); + expect(await create(first, "t-1", "doc-0", "k-0")).toMatchObject({ + kind: "applied", + }); + // Authorization, step 4, precedes the gate, step 7: the service actor holds a grant. + await admin.run(getFunctionName(internal.grants.grant), { + tenantId: "t-2", + principalKind: "service", + principalId: "svc-1", + permission: permissions.documents, + grantedBy: "native-test", + }); + await admin.setEnvironment({ MAINTENANCE_MODE: "restore" }); + const door = paused("write paused for all: restore"); + expect(await thrownData(create(first, "t-1", "doc-1", "k-1"))).toEqual(door); + expect(await thrownData(create(second, "t-2", "doc-1", "k-1"))).toEqual(door); + expect( + await thrownData( + admin.run( + getFunctionName(internal.depotCommands.createDocumentInternal), + { + tenantId: "t-2", + namespace: "worker", + actor: { kind: "service", id: "svc-1" }, + requestKey: "k-2", + input: { documentId: "doc-2", title: "Report" }, + }, + ), + ), + ).toEqual(door); + expect(await admin.run(getFunctionName(internal.gate.getGate), {})).toEqual({ + restore: true, + closed: [], + }); + expect(await admin.readTable("receipts")).toHaveLength(1); + for (const value of ["Restore", "off"]) { + await admin.setEnvironment({ MAINTENANCE_MODE: value }); + expect(await admin.run(getFunctionName(internal.gate.getGate), {})).toEqual( + { restore: false, closed: [] }, + ); + } + expect(await create(first, "t-1", "doc-1", "k-1")).toMatchObject({ + kind: "applied", + replayed: false, + }); + expect(await create(second, "t-2", "doc-1", "k-1")).toMatchObject({ + kind: "applied", + }); +}); + +// Optimistic concurrency makes the close a consistent cut: a command either commits before the close +// or reads the closed gate and is refused. What a client sees: every command ends applied or refused +// with writePaused, never with a technical failure; an applied command left its receipt and its event +// and a refused one neither; and every command sent after the close returned is refused. Which applied +// command committed before the close in the serial order is not observable from outside the backend. +test("native: a close racing commands in flight leaves each command applied with its receipt and event or refused with writePaused and nothing stored, and every command sent after the close is refused", async () => { + const backend = await fixtureBackend(); + const { admin } = backend; + const clients = await Promise.all( + [1, 2, 3, 4].map((n) => userIn(backend, "t-1", `user-${n}`)), + ); + let closed = false; + const outcomes: { + key: string; + sentAfterClose: boolean; + outcome: "applied" | "paused"; + }[] = []; + const writer = async (client: Client, n: number) => { + for (let i = 0; i < 12; i += 1) { + const key = `w${n}-${i}`; + const sentAfterClose = closed; + const outcome = await create(client, "t-1", key, key).then( + () => "applied" as const, + (error: unknown) => { + expect(error).toBeInstanceOf(ConvexError); + expect((error as ConvexError).data).toEqual( + paused("write paused for tenant:t-1: cut"), + ); + return "paused" as const; + }, + ); + outcomes.push({ key, sentAfterClose, outcome }); + } + }; + const close = (async () => { + while (outcomes.length < 6) await new Promise((r) => setTimeout(r, 5)); + await admin.run(getFunctionName(internal.gate.closeGate), { + scopeKey: "tenant:t-1", + reason: "cut", + operator: "ops-1", + }); + closed = true; + })(); + await Promise.all([...clients.map(writer), close]); + expect(outcomes).toHaveLength(48); + const applied = outcomes + .filter(({ outcome }) => outcome === "applied") + .map(({ key }) => key) + .sort(); + expect(applied.length).toBeGreaterThanOrEqual(6); + expect( + outcomes.filter( + ({ sentAfterClose, outcome }) => sentAfterClose && outcome === "applied", + ), + ).toEqual([]); + expect(outcomes.some(({ outcome }) => outcome === "paused")).toBe(true); + expect( + (await admin.readTable("receipts")) + .map(({ requestKey }) => requestKey) + .sort(), + ).toEqual(applied); + expect( + (await admin.readTable("events", { component: "depot" })) + .map(({ streamId }) => streamId) + .sort(), + ).toEqual(applied); + expect(await gateAudit(backend, "tenant:t-1")).toMatchObject([ + { kind: "gate.close", operator: "ops-1", reason: "cut" }, + ]); +}); diff --git a/tests/pure/gate.test.ts b/tests/pure/gate.test.ts new file mode 100644 index 0000000..78c4868 --- /dev/null +++ b/tests/pure/gate.test.ts @@ -0,0 +1,76 @@ +// The operator check, the scopes of a use case and the shapes of the gate and audit tables +// (spec:command.actor-and-scope, spec:application.write-pause, spec:operations.baseline-operations). +import { expect, test } from "vitest"; +import { auditTables } from "../../src/audit/index.js"; +import { assertOperator, limitOperatorBytes } from "../../src/command/index.js"; +import { + gateTables, + limitClosedScopes, + limitGateReasonBytes, + scopesOfUseCase, +} from "../../src/gate/index.js"; + +test("pure: assertOperator refuses an empty or whitespace-only operator and one above 512 bytes, and returns any other unchanged", () => { + for (const operator of ["", " ", "\t\n "]) + expect(() => assertOperator(operator)).toThrow( + "An operator entry needs a stated operator", + ); + expect(limitOperatorBytes).toBe(512); + expect(assertOperator("x".repeat(512))).toBe("x".repeat(512)); + expect(() => assertOperator("x".repeat(513))).toThrow( + "The stated operator is 513 bytes, above the limit of 512", + ); + // 256 two-byte characters are 512 bytes, and one more ASCII byte is 513. + expect(assertOperator("é".repeat(256))).toBe("é".repeat(256)); + expect(() => assertOperator("é".repeat(256) + "x")).toThrow( + "The stated operator is 513 bytes, above the limit of 512", + ); + expect(assertOperator(" ops-1 ")).toBe(" ops-1 "); +}); + +test("pure: scopesOfUseCase answers all, the tenant scope, then one source scope per stream type the declaration writes, in that order", () => { + expect( + scopesOfUseCase("t-1", [ + { contextId: "orders", streamType: "order" }, + { contextId: "inventory", streamType: "stockItem" }, + ]), + ).toEqual([ + "all", + "tenant:t-1", + "source:orders:order", + "source:inventory:stockItem", + ]); + expect(scopesOfUseCase("t-2", [])).toEqual(["all", "tenant:t-2"]); +}); + +test("pure: the gate's limits, its one index, and the audit indexes, which lead with the tenant except operatorAudit's", () => { + expect([limitClosedScopes, limitGateReasonBytes]).toEqual([16, 256]); + const indexes = (tables: Record unknown[] }>) => + Object.entries(tables).flatMap(([table, definition]) => + ( + definition[" indexes"]() as { + indexDescriptor: string; + fields: string[]; + }[] + ).map(({ indexDescriptor, fields }) => [table, indexDescriptor, fields]), + ); + expect(indexes(gateTables)).toEqual([ + ["maintenanceGates", "by_key", ["key"]], + ]); + expect(indexes(auditTables)).toEqual([ + ["auditRecords", "by_operation", ["tenantId", "operationId"]], + [ + "auditRecords", + "by_subject", + [ + "tenantId", + "subject.contextId", + "subject.streamType", + "subject.streamId", + "recordedAt", + ], + ], + ["auditRecords", "by_request_key", ["tenantId", "requestKey"]], + ["operatorAudit", "by_scope", ["scopeKey", "recordedAt"]], + ]); +}); diff --git a/tests/simulator/gate.test.ts b/tests/simulator/gate.test.ts new file mode 100644 index 0000000..bd8555c --- /dev/null +++ b/tests/simulator/gate.test.ts @@ -0,0 +1,683 @@ +// The maintenance gate, its operator entries and their audit records, the tenant list, and the audit +// record of the pipeline's step 10, on convex-test over the fixture composition +// (spec:application.write-pause, spec:operations.baseline-operations, spec:command.actor-and-scope). +import { convexTest } from "convex-test"; +import { defineSchema, defineTable } from "convex/server"; +import { ConvexError, v, type Value } from "convex/values"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { api, internal } from "../../fixture/convex/_generated/api.js"; +import annexSchema from "../../fixture/convex/annex/schema.js"; +import depotSchema from "../../fixture/convex/depot/schema.js"; +import { permissions } from "../../fixture/convex/depotCommands.js"; +import schema from "../../fixture/convex/schema.js"; +import { + insertGrant, + nextTenant, + revokeGrant, + runPipeline, + type CommandDeclaration, + type PipelineCall, +} from "../../src/command/index.js"; +import { gateAllows } from "../../src/gate/index.js"; +const { version } = JSON.parse( + readFileSync( + join(import.meta.dirname, "../../node_modules/convex-test/package.json"), + "utf8", + ), +) as { version: string }; +const name = (text: string) => `convex-test ${version}: ${text}`; +const modules = import.meta.glob("../../fixture/convex/**/*.ts"); +function app(appSchema: typeof schema = schema) { + const t = convexTest(appSchema, modules); + t.registerComponent( + "annex", + annexSchema, + import.meta.glob("../../fixture/convex/annex/**/*.ts"), + ); + t.registerComponent( + "depot", + depotSchema, + import.meta.glob("../../fixture/convex/depot/**/*.ts"), + ); + return t; +} +type App = ReturnType; +// The fixture's schema with an operatorAudit table whose validator refuses every gate change's record. +const refusingAudit = defineSchema({ + ...schema.tables, + operatorAudit: defineTable({ refused: v.literal(true) }), +}) as unknown as typeof schema; +const issuer = "https://fixture-issuer.test"; +async function caller(t: App, subject: string, tenantId = "t-1") { + await t.mutation(internal.grants.grant, { + tenantId, + principalKind: "human", + principalId: `${issuer}|${subject}`, + permission: permissions.documents, + grantedBy: "operator", + }); + return t.withIdentity({ issuer, subject }); +} +async function failure(promise: Promise): Promise { + return promise.then( + () => { + throw new Error("The call did not fail"); + }, + (error: unknown) => error, + ); +} +async function plainError(promise: Promise) { + const error = await failure(promise); + expect(error).not.toBeInstanceOf(ConvexError); + return String(error); +} +const gateDocuments = (t: App) => + t.run((ctx) => ctx.db.query("maintenanceGates").collect()); +const operatorAudit = (t: App) => + t.run((ctx) => ctx.db.query("operatorAudit").collect()); +const report = (documentId = "doc-1", tenantId = "t-1") => ({ + tenantId, + input: { documentId, title: "Report" }, +}); +const close = (t: App, scopeKey: string, reason = "maintenance") => + t.mutation(internal.gate.closeGate, { scopeKey, reason, operator: "ops-1" }); +const resume = (t: App, scopeKey: string) => + t.mutation(internal.gate.resumeGate, { scopeKey, operator: "ops-2" }); +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe("the gate read", () => { + test( + name( + "MAINTENANCE_MODE exactly restore answers closed for all with the reason restore and reads no document; other values leave the answer to the document", + ), + async () => { + const t = app(); + await close(t, "tenant:t-1", "move"); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + // A database that throws on any use: under the door, no document is read. + const untouchable = new Proxy( + {}, + { + get: () => { + throw new Error("The gate read touched the database"); + }, + }, + ) as Parameters[0]["db"]; + expect( + await gateAllows({ db: untouchable }, ["all", "tenant:t-2"]), + ).toEqual({ allowed: false, scopeKey: "all", reason: "restore" }); + for (const value of ["Restore", "restore ", "", "off"]) { + vi.stubEnv("MAINTENANCE_MODE", value); + expect( + await t.run((ctx) => gateAllows(ctx, ["all", "tenant:t-2"])), + ).toEqual({ allowed: true }); + expect( + await t.run((ctx) => gateAllows(ctx, ["all", "tenant:t-1"])), + ).toEqual({ allowed: false, scopeKey: "tenant:t-1", reason: "move" }); + } + vi.unstubAllEnvs(); + expect(await t.run((ctx) => gateAllows(ctx, ["all"]))).toEqual({ + allowed: true, + }); + }, + ); + + test( + name( + "an absent document is open and the read writes nothing; the first matching entry in array order answers", + ), + async () => { + const t = app(); + expect( + await t.run((ctx) => gateAllows(ctx, ["all", "tenant:t-1"])), + ).toEqual({ allowed: true }); + expect(await gateDocuments(t)).toEqual([]); + await close(t, "tenant:t-1", "first"); + await close(t, "all", "second"); + // The command's scopes are "all" before its tenant; the entry closed first answers. + expect( + await t.run((ctx) => gateAllows(ctx, ["all", "tenant:t-1"])), + ).toEqual({ allowed: false, scopeKey: "tenant:t-1", reason: "first" }); + }, + ); + + test( + name( + "an entry with no generation admits no one, and an entry naming a generation admits only a read stating that generation", + ), + async () => { + const t = app(); + const [first, second] = await t.run(async (ctx) => { + const row = { + readModel: "documentSummary", + projectionVersion: 1, + state: "building" as const, + pauseRequired: true, + batchSize: 1, + fence: 0, + batchesDone: 0, + rowsWritten: 0, + rowsSkipped: 0, + misses: 0, + startedAt: 0, + startedBy: { kind: "operator" as const, id: "ops-1" }, + updatedAt: 0, + }; + return [ + await ctx.db.insert("generations", { ...row, generation: 1 }), + await ctx.db.insert("generations", { ...row, generation: 2 }), + ]; + }); + await t.run((ctx) => + ctx.db.insert("maintenanceGates", { + key: "gates", + closed: [ + { + scopeKey: "source:depot:document", + reason: "rebuild", + generationId: first, + changedAt: 0, + changedBy: "ops-1", + }, + { + scopeKey: "tenant:t-1", + reason: "hold", + changedAt: 0, + changedBy: "ops-1", + }, + ], + updatedAt: 0, + }), + ); + const source = ["source:depot:document"]; + expect(await t.run((ctx) => gateAllows(ctx, source, first))).toEqual({ + allowed: true, + }); + expect(await t.run((ctx) => gateAllows(ctx, source, second))).toEqual({ + allowed: false, + scopeKey: "source:depot:document", + reason: "rebuild", + }); + expect(await t.run((ctx) => gateAllows(ctx, source))).toMatchObject({ + allowed: false, + }); + expect( + await t.run((ctx) => gateAllows(ctx, ["tenant:t-1"], first)), + ).toEqual({ allowed: false, scopeKey: "tenant:t-1", reason: "hold" }); + }, + ); +}); + +describe("a command in a closed scope", () => { + test( + name( + "a closed tenant refuses with exactly the writePaused data and stores nothing; another tenant writes; the same request key applies after the resume", + ), + async () => { + const t = app(); + const user = await caller(t, "user-1"); + const other = await caller(t, "user-2", "t-2"); + await close(t, "tenant:t-1", "move to the new region"); + const before = await t.run(async (ctx) => ({ + receipts: await ctx.db.query("receipts").collect(), + summaries: await ctx.db.query("documentSummaries").collect(), + audit: await ctx.db.query("auditRecords").collect(), + })); + const error = await failure( + user.mutation(api.depotCommands.createDocument, { + ...report(), + requestKey: "k-1", + }), + ); + expect(error).toBeInstanceOf(ConvexError); + expect((error as ConvexError).data).toEqual({ + kind: "transient", + code: "writePaused", + message: "write paused for tenant:t-1: move to the new region", + }); + expect( + await t.run(async (ctx) => ({ + receipts: await ctx.db.query("receipts").collect(), + summaries: await ctx.db.query("documentSummaries").collect(), + audit: await ctx.db.query("auditRecords").collect(), + })), + ).toEqual(before); + expect( + await other.mutation( + api.depotCommands.createDocument, + report("doc-1", "t-2"), + ), + ).toMatchObject({ kind: "applied" }); + await resume(t, "tenant:t-1"); + // The document is created at expected version 0, so no event of the refused command exists. + expect( + await user.mutation(api.depotCommands.createDocument, { + ...report(), + requestKey: "k-1", + }), + ).toMatchObject({ + kind: "applied", + replayed: false, + versions: [expect.objectContaining({ streamId: "doc-1", version: 1 })], + }); + }, + ); + + test( + name( + "under the restore door every command is refused with write paused for all: restore", + ), + async () => { + const t = app(); + const user = await caller(t, "user-1"); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + const error = await failure( + user.mutation(api.depotCommands.createDocument, report()), + ); + expect((error as ConvexError).data).toEqual({ + kind: "transient", + code: "writePaused", + message: "write paused for all: restore", + }); + }, + ); +}); + +describe("the operator entries", () => { + test( + name( + "close, double close, resume and resume of an open scope, each change with one operator audit record", + ), + async () => { + const t = app(); + expect(await close(t, "all", "release")).toBeNull(); + expect(await plainError(close(t, "all", "again"))).toContain( + "The scope all is already closed", + ); + const [gate] = await gateDocuments(t); + expect(gate?.closed).toEqual([ + { + scopeKey: "all", + reason: "release", + changedAt: expect.any(Number), + changedBy: "ops-1", + }, + ]); + expect(await resume(t, "all")).toBeNull(); + expect(await plainError(resume(t, "all"))).toContain( + "The scope all is not closed", + ); + expect(await t.query(internal.gate.getGate, {})).toEqual({ + restore: false, + closed: [], + }); + expect( + (await operatorAudit(t)).map( + ({ kind, scopeKey, reason, operator, generationId }) => ({ + kind, + scopeKey, + reason, + operator, + generationId, + }), + ), + ).toEqual([ + { + kind: "gate.close", + scopeKey: "all", + reason: "release", + operator: "ops-1", + generationId: undefined, + }, + { + kind: "gate.resume", + scopeKey: "all", + reason: "release", + operator: "ops-2", + generationId: undefined, + }, + ]); + }, + ); + + test( + name( + "the 17th scope is refused, and closeGate and resumeGate take only all or a tenant scope", + ), + async () => { + const t = app(); + for (let n = 1; n <= 16; n += 1) await close(t, `tenant:t-${n}`); + expect(await plainError(close(t, "tenant:t-17"))).toContain( + "The gate holds 16 closed scopes, its limit", + ); + for (const scopeKey of [ + "source:depot:document", + "tenant:", + `tenant:${"x".repeat(257)}`, + "read-model:documentSummary", + "ALL", + ]) { + expect(await plainError(close(t, scopeKey))).toContain( + `closeGate takes the scope all or a tenant scope, and ${scopeKey} is neither`, + ); + expect(await plainError(resume(t, scopeKey))).toContain( + `resumeGate takes the scope all or a tenant scope, and ${scopeKey} is neither`, + ); + } + expect(await close(t, "all").catch(String)).toContain("16 closed scopes"); + await resume(t, "tenant:t-16"); + expect(await close(t, `tenant:${"x".repeat(256)}`)).toBeNull(); + }, + ); + + test( + name( + "the reason is 1 to 256 bytes, and the operator is required, not only whitespace and at most 512 bytes, all before any read or write", + ), + async () => { + const t = app(); + for (const reason of ["", "é".repeat(129)]) + expect(await plainError(close(t, "all", reason))).toContain( + "The reason must be between 1 and 256 bytes", + ); + for (const operator of ["", " \t"]) { + expect( + await plainError( + t.mutation(internal.gate.closeGate, { + scopeKey: "all", + reason: "r", + operator, + }), + ), + ).toContain("An operator entry needs a stated operator"); + expect( + await plainError( + t.mutation(internal.gate.resumeGate, { scopeKey: "all", operator }), + ), + ).toContain("An operator entry needs a stated operator"); + } + // 171 three-byte characters are 513 bytes. + expect( + await plainError( + t.mutation(internal.gate.closeGate, { + scopeKey: "source:x:y", + reason: "", + operator: "€".repeat(171), + }), + ), + ).toContain("The stated operator is 513 bytes, above the limit of 512"); + expect(await gateDocuments(t)).toEqual([]); + expect(await operatorAudit(t)).toEqual([]); + expect(await close(t, "all", "é".repeat(128))).toBeNull(); + expect( + await t.mutation(internal.gate.resumeGate, { + scopeKey: "all", + operator: "€".repeat(170) + "xx", + }), + ).toBeNull(); + }, + ); + + test( + name( + "an operator audit insert that fails leaves the gate as it was on close and on resume, and no record", + ), + async () => { + const broken = app(refusingAudit); + expect(await plainError(close(broken, "all"))).not.toBe(""); + expect(await gateDocuments(broken)).toEqual([]); + const t = app(refusingAudit); + await t.run((ctx) => + ctx.db.insert("maintenanceGates", { + key: "gates", + closed: [ + { + scopeKey: "all", + reason: "release", + changedAt: 1, + changedBy: "ops-1", + }, + ], + updatedAt: 1, + }), + ); + const before = await gateDocuments(t); + expect(await plainError(resume(t, "all"))).not.toBe(""); + expect(await gateDocuments(t)).toEqual(before); + expect( + await t.run((ctx) => ctx.db.query("operatorAudit").collect()), + ).toEqual([]); + }, + ); + + test( + name( + "getGate answers restore and the closed entries, and getGateAudit pages one scope newest first at most 100 a page", + ), + async () => { + const t = app(); + expect(await t.query(internal.gate.getGate, {})).toEqual({ + restore: false, + closed: [], + }); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + expect(await t.query(internal.gate.getGate, {})).toEqual({ + restore: true, + closed: [], + }); + vi.unstubAllEnvs(); + for (let n = 0; n < 51; n += 1) { + vi.setSystemTime(1_000 + n); + await close(t, "tenant:t-1", `r-${n}`); + await resume(t, "tenant:t-1"); + } + vi.useRealTimers(); + await close(t, "tenant:t-2"); + const first = await t.query(internal.gate.getGateAudit, { + scopeKey: "tenant:t-1", + paginationOpts: { numItems: 500, cursor: null }, + }); + expect(first.page).toHaveLength(100); + expect(first.isDone).toBe(false); + expect(first.page[0]).toMatchObject({ + kind: "gate.resume", + reason: "r-50", + operator: "ops-2", + }); + const rest = await t.query(internal.gate.getGateAudit, { + scopeKey: "tenant:t-1", + paginationOpts: { numItems: 100, cursor: first.continueCursor }, + }); + expect(rest.page).toHaveLength(2); + expect(rest.page.at(-1)).toMatchObject({ + kind: "gate.close", + reason: "r-0", + }); + }, + ); +}); + +describe("the tenant list", () => { + test( + name( + "a first grant inserts the tenant's row, a second does not, revoking every grant keeps it, and nextTenant walks the rows in tenant-ID order", + ), + async () => { + const t = app(); + const grant = (tenantId: string, permission: string) => + t.run((ctx) => + insertGrant(ctx, { + tenantId, + principalKind: "service", + principalId: "svc-1", + permission, + grantedBy: "operator", + }), + ); + await grant("t-b", "one"); + await grant("t-b", "two"); + await grant("t-a", "one"); + const tenants = () => t.run((ctx) => ctx.db.query("tenants").collect()); + expect((await tenants()).map(({ tenantId }) => tenantId).sort()).toEqual([ + "t-a", + "t-b", + ]); + for (const permission of ["one", "two"]) + await t.run((ctx) => + revokeGrant(ctx, { + tenantId: "t-b", + principalKind: "service", + principalId: "svc-1", + permission, + }), + ); + expect(await tenants()).toHaveLength(2); + expect(await t.run((ctx) => nextTenant(ctx, null))).toBe("t-a"); + expect(await t.run((ctx) => nextTenant(ctx, "t-a"))).toBe("t-b"); + expect(await t.run((ctx) => nextTenant(ctx, "t-b"))).toBeNull(); + expect(await t.run((ctx) => nextTenant(ctx, "t-"))).toBe("t-a"); + }, + ); +}); + +describe("the audit record of step 10", () => { + const audited = ( + subjectFrom: boolean, + streamId: string | null, + ): CommandDeclaration<{ n: number }, { n: number }> => ({ + name: "Audited", + contractVersion: 1, + input: v.object({ n: v.number() }), + output: v.object({ n: v.number() }), + permission: { + permission: "probe", + ...(subjectFrom + ? { + subjectFrom: ({ n }) => ({ + contextId: "depot", + streamType: "document", + streamId: `from-input-${n}`, + }), + } + : {}), + }, + writes: [], + rejections: [], + audit: { kind: "security" }, + executor: async (_ctx, { tenantId, input }) => ({ + kind: "businessFailure", + result: input, + versions: + streamId === null + ? [] + : [ + { + tenantId, + contextId: "depot", + streamType: "document", + streamId, + version: 3, + }, + ], + streams: [], + }), + }); + const call: PipelineCall<{ n: number }> = { + tenantId: "t-1", + namespace: "worker", + actor: { kind: "service", id: "svc-1" }, + requestKey: "k-1", + causedBy: { + kind: "event", + tenantId: "t-1", + contextId: "depot", + eventId: "e-1", + }, + input: { n: 7 }, + }; + const grantProbe = (t: App) => + t.run((ctx) => + insertGrant(ctx, { + tenantId: "t-1", + principalKind: "service", + principalId: "svc-1", + permission: "probe", + grantedBy: "operator", + }), + ); + + test( + name( + "each field comes from its source, the subject from subjectFrom when declared and from the first version otherwise", + ), + async () => { + const t = app(); + await grantProbe(t); + const response = await t.run((ctx) => + runPipeline(ctx, audited(true, "from-version"), call), + ); + await t.run((ctx) => + runPipeline(ctx, audited(false, "from-version"), { + ...call, + requestKey: "k-2", + }), + ); + const records = await t.run((ctx) => + ctx.db.query("auditRecords").collect(), + ); + expect(records).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "t-1", + operationId: response.operationId, + requestKey: "k-1", + commandType: "Audited", + actor: { kind: "service", id: "svc-1" }, + subject: { + contextId: "depot", + streamType: "document", + streamId: "from-input-7", + }, + kind: "security", + decision: "businessFailure", + causedBy: call.causedBy, + recordedAt: expect.any(Number), + }, + expect.objectContaining({ + requestKey: "k-2", + subject: { + contextId: "depot", + streamType: "document", + streamId: "from-version", + }, + }), + ]); + }, + ); + + test( + name( + "a declaration that sets audit with no subjectFrom and no version fails closed, leaving no receipt", + ), + async () => { + const t = app(); + await grantProbe(t); + expect( + await plainError( + t.run((ctx) => runPipeline(ctx, audited(false, null), call)), + ), + ).toContain("its audit record has no subject"); + expect( + await t.run(async (ctx) => [ + ...(await ctx.db.query("receipts").collect()), + ...(await ctx.db.query("auditRecords").collect()), + ]), + ).toEqual([]); + }, + ); +}); diff --git a/tests/types/gate.test-d.ts b/tests/types/gate.test-d.ts new file mode 100644 index 0000000..95bc584 --- /dev/null +++ b/tests/types/gate.test-d.ts @@ -0,0 +1,51 @@ +// The compiled shapes of the audit record input and of the gate's operator entries +// (spec:operations.baseline-operations, spec:application.write-pause). +import type { + FunctionArgs, + FunctionReturnType, + WithoutSystemFields, +} from "convex/server"; +import { expectTypeOf, test } from "vitest"; +import { internal } from "../../fixture/convex/_generated/api.js"; +import type { + AuditRecord, + AuditRecordInput, + OperatorAuditInput, + OperatorAuditRecord, +} from "../../src/audit/index.js"; +import type { ClosedEntry } from "../../src/gate/index.js"; + +test("AuditRecordInput is the auditRecords document without recordedAt", () => { + expectTypeOf().toEqualTypeOf< + Omit, "recordedAt"> + >(); + expectTypeOf().toEqualTypeOf< + Omit, "recordedAt"> + >(); +}); + +test("the operator entries take and return what the write pause pins", () => { + expectTypeOf>().toEqualTypeOf<{ + scopeKey: string; + reason: string; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf<{ + scopeKey: string; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + // getGate takes no argument: its args validator is the empty object. + expectTypeOf>().toBeNever(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf<{ restore: boolean; closed: ClosedEntry[] }>(); + expectTypeOf< + FunctionReturnType["page"][number] + >().toEqualTypeOf(); +}); From 51c89f370270f293e71f1b8caefee92f64fad8b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:51:02 +0200 Subject: [PATCH 038/155] design: a delegated row for narrowing a command's grant read, with the measured cost Co-Authored-By: Claude Opus 5.5 --- design/decisions/register.json | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/design/decisions/register.json b/design/decisions/register.json index ee26cdd..f1f2bf2 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -6272,6 +6272,41 @@ "decidedBy": "owner", "decidedOn": "2026-10-02", "ruling": "Sc L2-9 reads as the local observation: the production composition on the pinned local backend, with the same authority, schemas, concurrency and code path a release has. No Spec claims parity with a hosted deployment. The concurrency half is the contention run on the production composition, as OD-007's checked lean says. A hosted run stays deferred with Probe 3's quota half (OD-036). Ruled by the main thread on the owner's words of 2026-10-02 (\"Decide on the forks in the roadmap.\"), and the owner agreed the same day to record the rulings as the owner's (\"i agree with the least noisy and cleanest path forward.\")." + }, + { + "id": "OD-154", + "title": "Whether a command's grant read narrows from every grant of the principal to the grants of the command's permission", + "sources": [ + { + "kind": "specLine", + "ref": "command.tenancy-and-authority#limitGrantsPerPrincipal", + "file": "design/specs/command/tenancy-and-authority.sdp.md", + "line": 78 + } + ], + "families": [ + "command" + ], + "cites": [ + "F13", + "E-37" + ], + "provisionalReading": "at most 500 grants read per principal per tenant per command; the read takes 501 and a 501st row throws a plain `Error`, a technical failure, never a silent truncation", + "blocks": null, + "advisor": "convex", + "status": "sorted", + "foldedInto": [], + "class": "delegated", + "reason": "Rule 3: what a command reads, settled by a measurement on the native backend and a ruling on the 500-grant refusal that lead 20 of design/STATE.md holds; raised by the measurement of PlaceOrder and made a row by the main thread's ruling on it.", + "lean": { + "do": "Keep the grant read at every grant of the principal until the 500-grant refusal is ruled, then narrow it with an index on tenant, principal and permission.", + "restsOn": "Measured at the native backend tier on the pinned local backend: on the ten-line PlaceOrder the three grants of other permissions cost 3 documents and 918 bytes, 8.57 percent of the documents and 6.35 percent of the bytes read; narrowing the read changes the refusal at more than 500 grants of a principal, which lead 20 of design/STATE.md holds [design/specs/command/tenancy-and-authority.sdp.md:78]." + }, + "checkedBy": null, + "fork": null, + "decidedBy": null, + "decidedOn": null, + "ruling": null } ], "possiblySame": [ From 24e208ddd87907035397107fc0ef02db53335b80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:51:17 +0200 Subject: [PATCH 039/155] design: the rebuild's batch risk and open question say what the migrations probe showed Co-Authored-By: Claude Opus 5.5 --- design/specs/application/rebuild.sdp.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 1fbdd28..81340c2 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -46,7 +46,7 @@ The registry, the progress row, the per-entity marker and the gate document are - value: The common rebuild costs no downtime and no projection job; the write pause is paid only where a consistent cut is needed (D9) - risk: The standing cost is a generation registry read in every command that maintains a read model, a generation on every row, the earlier projection version kept in the code while its generation can be rolled back to, the batch chain, and for the write pause a gate every writer reads (D9) - risk: A pass costs at least one batch per tenant of the tenant list, because a batch names one tenant, so a deployment of many empty tenants pays one scheduled mutation for each (E-49, Law 11) -- risk: The batch bound is a design choice under the transaction limits until Probe 6 shows how a batch races a live command (F13, Probe 6) +- risk: The batch bound is a design choice under the transaction limits and no question of correctness: a batch that races a live command on one row is ordered by optimistic concurrency and the row ends at the source's newest stream version in either commit order, and a batch that reads past the read limit fails as a whole and leaves its checkpoint as it was (F13, F1, F17, Probe 6) - assumption: Mutations are serializable under optimistic concurrency, so a backfill batch and a live command that touch the same row serialize and the loser re-runs (F1) - assumption: A scheduled mutation that loses an optimistic-concurrency race is run again by Convex until it commits; the pinned packages do not show a bound on those runs, the design needs none for correctness, and Probe 6 shows what a stalled batch looks like (F9, F1, Probe 6) - assumption: A parent query calling a component query stays reactive, so switching the active generation in the registry re-runs every subscription that reads it, and the enumeration's `paginator` page relayed by the parent stays contiguous under live writes; this is F15, and Probe 5 showed both on the pinned backend (F15, Probe 5, S4) @@ -55,7 +55,7 @@ The registry, the progress row, the per-entity marker and the gate document are ### Open questions -- [non-blocking] Probe 6 pending: a backfill batch racing a live command under optimistic concurrency, and whether `@convex-dev/migrations` can drive a batch from inside or across a component boundary (Probe 6, F1, F17, D9) +- [non-blocking] `@convex-dev/migrations` walks only the tables of the component it is defined in, reaches a context's rows only through a parent internal mutation written by hand against a contract its README does not document and the compiled tier needs a cast for, keeps a status apart from the generation's state, and leaves no error on its row when a batch fails past the read limit, so every batch is driven by the rebuild's own self-scheduled internal mutation and the progress row is the one checkpoint; a hosted deployment and sustained contention are not observed (Probe 6, F1, F17, D9) - [non-blocking] Extension E-40: the doc binds resumable batching to `@convex-dev/migrations` and gives no batch size; the component iterates a parent table while the sources of a read model live in a context component. The option taken here: a batch is one mutation with one cursor on the progress row and a bound of at most 100 entities, capped by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget; every batch, the backfill and the verify pass over a context's enumeration and the purge over a parent table, is driven by the platform's own self-scheduled internal mutation, and the rebuild uses the component for nothing, because Probe 6 showed that it walks only the tables of the component it is defined in and drives a context's rows only through a parent internal mutation written by hand against a contract its README does not document; the progress row is the one checkpoint (E-40, E-24, E-9, F17, F13, F1) - [non-blocking] Extension E-8: the registry states, the rollback period, what a rollback does, the gate document, the scope a paused rebuild closes and the reach of a generation are design choices the doc does not make; they are pinned in the two contracts under this workflow (E-8, E-9, E-38, D9) - [non-blocking] Extension E-8, what this workflow does not offer: the online path is pinned for a per-entity read model with one source stream type; an `AggregateProjection` and a row built from more than one source stream type have no rebuild here, and each gets one when a view needs it; the paused path of a history view is given below in outline, its cursor across several sources and its verify pass are that view's own rules, and `rollbackGeneration` refuses a generation that needs a write pause, because nothing would hold its sources while live commands fold onto a row that missed the events since its retirement (E-8, E-9, D9, D18) From cb54d209ce8af42f3bbaa5edb875f0c8f89e692f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:52:01 +0200 Subject: [PATCH 040/155] design: the package counts recomputed from the tree, the contention example in package D Co-Authored-By: Claude Opus 5.5 --- design/README.md | 4 ++-- design/specs/command-pipeline.pack.sdp.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/design/README.md b/design/README.md index 09e298a..5e5717e 100644 --- a/design/README.md +++ b/design/README.md @@ -338,7 +338,7 @@ The Layer 2 design added two more for the same reason, and nine examples that ve | none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-operator-cancel` | Probe 6: an operator cancels a committed successor | | none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-transaction-failure` | Probe 6: a transaction failure preserves the saved cursor | -Package A therefore has 93 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. +Package A therefore has 93 Specs, Package C has 17 examples, Package D has 26, and the corpus has 60 examples that bind a doc row. The example domain's second lifecycle command added six examples that verify no doc row, each a child of `spec:application.orders-inventory-example` under E-46: @@ -351,7 +351,7 @@ The example domain's second lifecycle command added six examples that verify no | none | `spec:application.orders-inventory-example.cancel-second-context-rejects` | verifies that a release Inventory refuses rolls back the cancel Orders recorded | | none | `spec:application.orders-inventory-example.cancel-without-grant-refused` | verifies that `CancelOrder` requires its own permission | -Package D therefore has 25 examples. +Package D therefore has 26 examples. ## Integration notes diff --git a/design/specs/command-pipeline.pack.sdp.md b/design/specs/command-pipeline.pack.sdp.md index 6e98a0c..83a757a 100644 --- a/design/specs/command-pipeline.pack.sdp.md +++ b/design/specs/command-pipeline.pack.sdp.md @@ -32,4 +32,4 @@ modelRefs: Package C. Layer 1's parent-side path: the one-mutation command pipeline, the outcome and rejection boundary, idempotency receipts and their table, tenancy and authority, the actor and scope contract, and the command declaration, with the Layer 1 scenarios those components own. -Membership order follows the plan's inventory, parents before children: the pipeline and its three L1-2 injection points, the outcome boundary and L1-12, receipts with L1-3 as two cases, L1-4, L1-5 and L1-9 as two cases, the receipts table, tenancy with L1-6, L1-7 as two cases and L1-8, the actor and scope contract, and the declaration with L2-4 as two cases. Seven Specs and sixteen examples. +Membership order follows the plan's inventory, parents before children: the pipeline and its three L1-2 injection points, the outcome boundary and L1-12, receipts with L1-3 as two cases, L1-4, L1-5 and L1-9 as two cases, the receipts table, tenancy with L1-6, L1-7 as two cases and L1-8, the actor and scope contract, and the declaration with L2-4 as two cases. Seven Specs and seventeen examples. From f7113570b991ca827917caec805575c5810944ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:55:19 +0200 Subject: [PATCH 041/155] Test maintenance gate and audit contracts --- tests/native/gate.test.ts | 321 ++++++++++++++ tests/pure/gate-handlers.ts | 34 ++ tests/pure/gate.test.ts | 186 +++++++++ tests/simulator/audit.test.ts | 329 +++++++++++++++ tests/simulator/gate-inspection.ts | 10 + tests/simulator/gate-pipeline.test.ts | 183 ++++++++ tests/simulator/gate-support.ts | 122 ++++++ tests/simulator/gate.test.ts | 577 ++++++++++++++++++++++++++ tests/simulator/tenants.test.ts | 80 ++++ tests/types/gate-audit.test-d.ts | 173 ++++++++ 10 files changed, 2015 insertions(+) create mode 100644 tests/native/gate.test.ts create mode 100644 tests/pure/gate-handlers.ts create mode 100644 tests/pure/gate.test.ts create mode 100644 tests/simulator/audit.test.ts create mode 100644 tests/simulator/gate-inspection.ts create mode 100644 tests/simulator/gate-pipeline.test.ts create mode 100644 tests/simulator/gate-support.ts create mode 100644 tests/simulator/gate.test.ts create mode 100644 tests/simulator/tenants.test.ts create mode 100644 tests/types/gate-audit.test-d.ts diff --git a/tests/native/gate.test.ts b/tests/native/gate.test.ts new file mode 100644 index 0000000..a97c687 --- /dev/null +++ b/tests/native/gate.test.ts @@ -0,0 +1,321 @@ +import { + getFunctionName, + makeFunctionReference, + type PaginationResult, +} from "convex/server"; +import { ConvexError, type Value } from "convex/values"; +import { expect, test } from "vitest"; +import { api, internal } from "../../fixture/convex/_generated/api.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend, measure } from "../../harness/native.js"; +import type { Backend } from "../../harness/backend.js"; +import type { CompletionRecord } from "../../harness/admin.js"; + +const close = (backend: Backend, scopeKey = "tenant:t") => + backend.admin.run(getFunctionName(internal.gate.closeGate), { + scopeKey, + reason: "repair", + operator: "closer", + }); +const resume = (backend: Backend, scopeKey = "tenant:t") => + backend.admin.run(getFunctionName(internal.gate.resumeGate), { + scopeKey, + operator: "resumer", + }); +async function clientFor(backend: Backend, tenantId = "t") { + const subject = `caller-${tenantId}`; + for (const permission of ["depot.documents", "depot.stock", "depot.orders"]) + await backend.admin.run(getFunctionName(internal.grants.grant), { + tenantId, + principalKind: "human", + principalId: `${backend.issuer.issuer}|${subject}`, + permission, + grantedBy: "setup", + }); + return ordinaryClient(backend.url, { + token: await backend.issuer.token(subject), + }); +} +async function expectPaused( + promise: Promise, + scopeKey: string, + reason: string, +) { + const error: unknown = await promise.then( + () => { + throw new Error("Expected a write pause"); + }, + (error: unknown) => error, + ); + expect(error).toBeInstanceOf(ConvexError); + expect((error as ConvexError).data).toStrictEqual({ + kind: "transient", + code: "writePaused", + message: `write paused for ${scopeKey}: ${reason}`, + }); +} +const document = (tenantId = "t", documentId = "d") => ({ + tenantId, + requestKey: `key-${documentId}`, + input: { documentId, title: "Document" }, +}); +async function stored(backend: Backend) { + return { + receipts: await backend.admin.readTable("receipts"), + events: await backend.admin.readTable("events", { component: "depot" }), + streams: await backend.admin.readTable("streams", { component: "depot" }), + rows: await backend.admin.readTable("documentSummaries"), + audits: await backend.admin.readTable("auditRecords"), + }; +} + +// spec:application.write-pause errorCodeWritePaused, fnCloseGate, fnResumeGate, fnGetGateAudit; spec:operations.baseline-operations tableOperatorAudit. +test("native: a closed tenant refuses a public command, another tenant writes, and each operator change is readable through getGateAudit", async () => { + const backend = await fixtureBackend(); + const first = await clientFor(backend); + const other = await clientFor(backend, "other"); + await close(backend); + const before = await stored(backend); + await expectPaused( + first.mutation(api.depotCommands.createDocument, document()), + "tenant:t", + "repair", + ); + expect(await stored(backend)).toEqual(before); + expect( + await other.mutation(api.depotCommands.createDocument, document("other")), + ).toMatchObject({ kind: "applied", replayed: false }); + await resume(backend); + expect( + await first.mutation(api.depotCommands.createDocument, document()), + ).toMatchObject({ kind: "applied", replayed: false }); + const audit = (await backend.admin.run( + getFunctionName(internal.gate.getGateAudit), + { scopeKey: "tenant:t", paginationOpts: { numItems: 100, cursor: null } }, + )) as unknown as PaginationResult>; + expect(audit.isDone).toBe(true); + expect(audit.page).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + kind: "gate.resume", + scopeKey: "tenant:t", + reason: "repair", + operator: "resumer", + recordedAt: expect.any(Number), + }, + { + _id: expect.any(String), + _creationTime: expect.any(Number), + kind: "gate.close", + scopeKey: "tenant:t", + reason: "repair", + operator: "closer", + recordedAt: expect.any(Number), + }, + ]); +}); + +// spec:application.write-pause restoreDoor, errorCodeWritePaused, Contract "The restore door is read before the gate document"; spec:command.command-pipeline gateRead. +test("native: setting restore on the running backend refuses every declared command route without a deploy and clearing it admits the same keys", async () => { + const backend = await fixtureBackend(); + const client = await clientFor(backend); + await backend.admin.run(getFunctionName(internal.readModels.activate), { + readModel: "documentSummary", + startedBy: { kind: "operator", id: "setup" }, + }); + type Write = { + name: string; + input: Record; + internal: boolean; + requestKey: string; + }; + const writes: Write[] = []; + for (const internalRoute of [false, true]) { + const suffix = internalRoute ? "internal" : "public"; + for (const id of [`submit-${suffix}`, `ship-${suffix}`, `amend-${suffix}`]) + await client.mutation( + api.depotCommands.createDocument, + document("t", id), + ); + await client.mutation(api.depotCommands.submitDocument, { + tenantId: "t", + input: { documentId: `ship-${suffix}`, expectedVersion: 1 }, + }); + await client.mutation(api.depotCommands.addStock, { + tenantId: "t", + input: { lines: [{ productId: `stock-${suffix}`, quantity: 10 }] }, + }); + const cases: [string, Record][] = [ + [ + "depotCommands:createDocument", + { documentId: `create-${suffix}`, title: "Document" }, + ], + [ + "depotCommands:createSummarizedDocument", + { documentId: `summary-${suffix}`, title: "Document" }, + ], + [ + "depotCommands:submitDocument", + { documentId: `submit-${suffix}`, expectedVersion: 1 }, + ], + [ + "depotCommands:shipDocument", + { documentId: `ship-${suffix}`, expectedVersion: 2 }, + ], + [ + "depotCommands:amendDocument", + { documentId: `amend-${suffix}`, title: "Changed", expectedVersion: 1 }, + ], + [ + "depotCommands:registerDocument", + { + documentId: `register-${suffix}`, + reference: `reference-${suffix}`, + title: "Document", + }, + ], + [ + "depotCommands:addStock", + { lines: [{ productId: `added-${suffix}`, quantity: 1 }] }, + ], + [ + "depotCommands:claimStock", + { lines: [{ productId: `stock-${suffix}`, quantity: 1 }] }, + ], + [ + "orders:placeOrder", + { + orderId: `order-${suffix}`, + title: "Order", + lines: [{ productId: `stock-${suffix}`, quantity: 1 }], + }, + ], + ]; + for (const [name, input] of cases) + writes.push({ + name: name + (internalRoute ? "Internal" : ""), + input, + internal: internalRoute, + requestKey: `${suffix}-${name}`, + }); + } + function send(write: Write) { + const args = { + tenantId: "t", + input: write.input, + requestKey: write.requestKey, + }; + return write.internal + ? backend.admin.run(write.name, { + ...args, + namespace: "worker", + actor: { kind: "human", id: `${backend.issuer.issuer}|caller-t` }, + }) + : client.mutation(makeFunctionReference<"mutation">(write.name), args); + } + await backend.admin.setEnvironment({ MAINTENANCE_MODE: "restore" }); + const before = await stored(backend); + for (const write of writes) await expectPaused(send(write), "all", "restore"); + expect( + await backend.admin.run(getFunctionName(internal.gate.getGate), {}), + ).toEqual({ restore: true, closed: [] }); + // Even the command whose decider always fails must be stopped before it executes. + await expectPaused( + client.mutation(api.depotCommands.failIfDecided, { + tenantId: "t", + input: { documentId: "amend-public", expectedVersion: 1 }, + }), + "all", + "restore", + ); + expect(await stored(backend)).toEqual(before); + await backend.admin.setEnvironment({ MAINTENANCE_MODE: "" }); + expect( + await backend.admin.run(getFunctionName(internal.gate.getGate), {}), + ).toEqual({ restore: false, closed: [] }); + for (const write of writes) + expect(await send(write)).toMatchObject({ + kind: "applied", + replayed: false, + }); +}); + +// spec:application.write-pause occSerialization, Contract "Background writers are fenced, not drained". +test("native: two independent writers race a close and no successful command completes beyond the close", async () => { + const backend = await fixtureBackend(); + const one = await clientFor(backend); + const two = ordinaryClient(backend.url, { + token: await backend.issuer.token("caller-t"), + }); + const identifier = getFunctionName(api.depotCommands.createDocument); + const closeIdentifier = getFunctionName(internal.gate.closeGate); + let overlapping = 0; + for (let trial = 0; trial < 8; trial++) { + const mark = await backend.admin.logMark(); + const settled = await Promise.allSettled([ + one.mutation( + api.depotCommands.createDocument, + document("t", `one-${trial}`), + ), + two.mutation( + api.depotCommands.createDocument, + document("t", `two-${trial}`), + ), + close(backend), + ]); + expect(settled[2]?.status).toBe("fulfilled"); + for (const result of settled.slice(0, 2)) { + if (result.status === "fulfilled") + expect(result.value).toMatchObject({ kind: "applied" }); + else + await expectPaused(Promise.reject(result.reason), "tenant:t", "repair"); + } + const records = await backend.admin.completionsSince( + mark, + (rows) => + rows.filter( + (row) => + row.componentPath === null && + (row.identifier === identifier || + row.identifier === closeIdentifier), + ).length >= 3, + ); + const closeRecord = records.find( + (row) => row.identifier === closeIdentifier && row.componentPath === null, + ); + expect(closeRecord).toBeDefined(); + const commands = records.filter( + (row) => row.identifier === identifier && row.componentPath === null, + ); + expect(commands).toHaveLength(2); + for (const command of commands) { + if (command.error === null) + expect(command.timestamp).toBeLessThanOrEqual(closeRecord!.timestamp); + if (overlap(command, closeRecord!)) overlapping++; + } + const cut = await stored(backend); + await expectPaused( + one.mutation( + api.depotCommands.createDocument, + document("t", `after-${trial}`), + ), + "tenant:t", + "repair", + ); + expect(await stored(backend)).toEqual(cut); + await resume(backend); + } + measure( + "commands overlapping the gate close by completion interval", + overlapping, + ); + // Without an overlapping invocation this run cannot witness the concurrency obligation. + expect(overlapping).toBeGreaterThan(0); +}); +function overlap(a: CompletionRecord, b: CompletionRecord) { + return ( + a.timestamp - a.executionTime < b.timestamp && + b.timestamp - b.executionTime < a.timestamp + ); +} diff --git a/tests/pure/gate-handlers.ts b/tests/pure/gate-handlers.ts new file mode 100644 index 0000000..efdeace --- /dev/null +++ b/tests/pure/gate-handlers.ts @@ -0,0 +1,34 @@ +import type { + GenericDataModel, + GenericMutationCtx, + GenericQueryCtx, +} from "convex/server"; +import { + closeGate, + resumeGate, + getGate, + getGateAudit, +} from "../../src/gate/index.js"; + +// Convex exposes _handler at runtime but omits it from RegisteredMutation's public type. +// Calling it lets a pure test stop at the first database access. +export function gateHandler(entry: "closeGate" | "resumeGate") { + return (entry === "closeGate" ? closeGate : resumeGate) as unknown as { + _handler: ( + ctx: GenericMutationCtx, + args: { scopeKey: string; operator: string; reason?: string }, + ) => Promise; + }; +} + +export function gateQueryHandler(entry: "getGate" | "getGateAudit") { + return (entry === "getGate" ? getGate : getGateAudit) as unknown as { + _handler: ( + ctx: GenericQueryCtx, + args: { + scopeKey?: string; + paginationOpts?: { numItems: number; cursor: string | null }; + }, + ) => Promise; + }; +} diff --git a/tests/pure/gate.test.ts b/tests/pure/gate.test.ts new file mode 100644 index 0000000..516c4db --- /dev/null +++ b/tests/pure/gate.test.ts @@ -0,0 +1,186 @@ +import { + paginationOptsValidator, + paginationResultValidator, + type GenericDataModel, + type GenericMutationCtx, +} from "convex/server"; +import { v } from "convex/values"; +import { expect, test, vi } from "vitest"; +import { assertOperator, limitOperatorBytes } from "../../src/command/index.js"; +import { + closeGate, + resumeGate, + getGate, + getGateAudit, + scopesOfUseCase, +} from "../../src/gate/index.js"; +import { gateHandler } from "./gate-handlers.js"; + +// spec:command.actor-and-scope fnAssertOperator, limitOperatorBytes. +test.each(["", " ", "\t\n", "\u2003", " ".repeat(512), " ".repeat(513)])( + "pure: an empty or whitespace operator %j is refused", + (operator) => { + expect(() => assertOperator(operator)).toThrow( + new Error("An operator entry needs a stated operator"), + ); + }, +); + +// spec:command.actor-and-scope fnAssertOperator, limitOperatorBytes. +test.each(["a", " a ", "a".repeat(512), "é".repeat(256), "😀".repeat(128)])( + "pure: a bounded operator is returned unchanged, %j", + (operator) => { + expect(limitOperatorBytes).toBe(512); + expect(assertOperator(operator)).toBe(operator); + }, +); + +// spec:command.actor-and-scope fnAssertOperator. +test.each(["a".repeat(513), "é".repeat(256) + "a", "😀".repeat(128) + "é"])( + "pure: an operator over the UTF-8 bound is refused, %j", + (operator) => { + const bytes = new TextEncoder().encode(operator).length; + expect(() => assertOperator(operator)).toThrow( + new Error( + `The stated operator is ${bytes} bytes, above the limit of 512`, + ), + ); + }, +); + +// spec:application.write-pause fnScopesOfUseCase, typeScopeKey. +test("pure: scopes preserve declaration order and the whole tenant ID", () => { + const writes = [ + { contextId: "depot", streamType: "stock" }, + { contextId: "depot", streamType: "document" }, + { contextId: "depot", streamType: "stock" }, + ]; + expect(scopesOfUseCase("source:depot:stock", writes)).toEqual([ + "all", + "tenant:source:depot:stock", + "source:depot:stock", + "source:depot:document", + "source:depot:stock", + ]); + expect(scopesOfUseCase("t", [])).toEqual(["all", "tenant:t"]); +}); + +for (const entry of ["closeGate", "resumeGate"] as const) { + // spec:application.write-pause fnCloseGate, fnResumeGate. + test.each([ + "", + "tenant:", + "source:depot:document", + "All", + "tenant:" + "a".repeat(257), + "tenant:" + "é".repeat(129), + ])( + `pure: ${entry} refuses invalid scope %j before a read`, + async (scopeKey) => { + const access = vi.fn(() => { + throw new Error("Database accessed"); + }); + const ctx = new Proxy( + {}, + { get: access }, + ) as GenericMutationCtx; + await expect( + gateHandler(entry)._handler(ctx, { + scopeKey, + reason: "work", + operator: "op", + }), + ).rejects.toThrow( + new Error( + `${entry} takes the scope all or a tenant scope, and ${scopeKey} is neither`, + ), + ); + expect(access).not.toHaveBeenCalled(); + }, + ); + + // spec:application.write-pause fnCloseGate, fnResumeGate, typeScopeKey. + test.each([ + "all", + "tenant:t", + "tenant: ", + "tenant:source:depot:document", + "tenant:" + "a".repeat(256), + "tenant:" + "é".repeat(128), + ])(`pure: ${entry} accepts the scope syntax %j`, async (scopeKey) => { + const access = vi.fn(() => { + throw new Error("Reached database"); + }); + const ctx = { + db: new Proxy({}, { get: access }), + } as GenericMutationCtx; + await expect( + gateHandler(entry)._handler(ctx, { + scopeKey, + reason: "work", + operator: "op", + }), + ).rejects.toThrow(new Error("Reached database")); + expect(access).toHaveBeenCalled(); + }); +} + +// spec:application.write-pause fnCloseGate, fnResumeGate, fnGetGate, fnGetGateAudit, validatorClosedEntry. +test("pure: registered gate entries export the exact argument and return validators", () => { + const closed = v.object({ + scopeKey: v.string(), + reason: v.string(), + generationId: v.optional(v.id("generations")), + changedAt: v.number(), + changedBy: v.string(), + }); + const audit = v.object({ + _id: v.id("operatorAudit"), + _creationTime: v.number(), + kind: v.union(v.literal("gate.close"), v.literal("gate.resume")), + scopeKey: v.string(), + reason: v.string(), + generationId: v.optional(v.id("generations")), + operator: v.string(), + recordedAt: v.number(), + }); + const cases = [ + [ + closeGate, + v.object({ + scopeKey: v.string(), + reason: v.string(), + operator: v.string(), + }), + v.null(), + ], + [ + resumeGate, + v.object({ scopeKey: v.string(), operator: v.string() }), + v.null(), + ], + [ + getGate, + v.object({}), + v.object({ restore: v.boolean(), closed: v.array(closed) }), + ], + [ + getGateAudit, + v.object({ + scopeKey: v.string(), + paginationOpts: paginationOptsValidator, + }), + paginationResultValidator(audit), + ], + ] as const; + for (const [entry, args, returns] of cases) { + const exported = entry as unknown as { + exportArgs(): string; + exportReturns(): string; + isInternal: boolean; + }; + expect(exported.isInternal).toBe(true); + expect(JSON.parse(exported.exportArgs())).toEqual(args.json); + expect(JSON.parse(exported.exportReturns())).toEqual(returns.json); + } +}); diff --git a/tests/simulator/audit.test.ts b/tests/simulator/audit.test.ts new file mode 100644 index 0000000..f7c9f2b --- /dev/null +++ b/tests/simulator/audit.test.ts @@ -0,0 +1,329 @@ +import { ConvexError, v } from "convex/values"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { components } from "../../fixture/convex/_generated/api.js"; +import { + writeAudit, + writeOperatorAudit, + type AuditRecordInput, +} from "../../src/audit/index.js"; +import { + insertGrant, + runPipeline, + type CommandDeclaration, + type PipelineCall, + type SubjectRef, +} from "../../src/command/index.js"; +import type { OperationRef } from "../../src/context/index.js"; +import { + gateApp, + caught, + generationIds, + depotRows, + type GateApp, +} from "./gate-support.js"; + +beforeEach(() => { + vi.stubEnv("MAINTENANCE_MODE", undefined); + vi.spyOn(Date, "now").mockReturnValue(1_000_000); +}); +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); +const actor = { + kind: "service", + id: "service-id", + issuer: "issuer", + onBehalfOf: { kind: "human", id: "delegator" }, + delegationRef: "delegation", +} as const; +const explicitCause = { + kind: "event", + tenantId: "cause-tenant", + contextId: "cause-context", + eventId: "cause-event", +} as const; +const subject = { contextId: "depot", streamType: "document", streamId: "d" }; +const auditInput: AuditRecordInput = { + tenantId: "t", + operationId: "operation", + requestKey: "request", + commandType: "AuditedDocument", + actor, + subject, + kind: "security", + decision: "applied", + causedBy: explicitCause, +}; +const call: PipelineCall<{ documentId: string; title: string }> = { + tenantId: "t", + namespace: "worker", + actor, + requestKey: "request-key", + correlationId: "correlation-is-not-causation", + causedBy: explicitCause, + input: { documentId: "d", title: "Document" }, +}; +const grant = (t: GateApp) => + t.run((ctx) => + insertGrant(ctx, { + tenantId: "t", + principalKind: actor.kind, + principalId: actor.id, + permission: "documents", + grantedBy: "admin", + }), + ); +function declaration( + options: { + kind?: "security" | "business"; + decision?: "applied" | "businessFailure"; + subjectFrom?: (input: typeof call.input) => SubjectRef; + versions?: boolean; + } = {}, +) { + const seen: OperationRef[] = []; + const decl: CommandDeclaration = { + name: "AuditedDocument", + contractVersion: 1, + input: v.object({ documentId: v.string(), title: v.string() }), + output: v.null(), + permission: { + permission: "documents", + ...(options.subjectFrom ? { subjectFrom: options.subjectFrom } : {}), + }, + audit: { kind: options.kind ?? "business" }, + writes: [{ contextId: "depot", streamType: "document" }], + rejections: ["titleRequired"], + executor: async ( + ctx, + { tenantId, actor: established, operation, input }, + ) => { + seen.push(operation); + const result = await ctx.runMutation( + components.depot.operations.createDocuments, + { + tenantId, + actor: established, + operation, + input: { + documents: [input, { documentId: "second", title: "Second" }], + }, + }, + ); + return { + ...result, + kind: options.decision ?? "applied", + result: null, + versions: options.versions === false ? [] : result.versions, + }; + }, + }; + return { decl, seen }; +} + +// spec:operations.baseline-operations fnWriteAudit, typeAuditRecordInput, tableAuditRecords. +test("convex-test: writeAudit preserves every input field and sets recordedAt to the mutation time", async () => { + const t = gateApp(); + const result = await t.run(async (ctx) => ({ + at: Date.now(), + id: await writeAudit(ctx, auditInput), + })); + expect(await t.run((ctx) => ctx.db.get(result.id))).toEqual({ + ...auditInput, + _id: result.id, + _creationTime: expect.any(Number), + recordedAt: result.at, + }); +}); + +// spec:operations.baseline-operations fnWriteOperatorAudit, tableOperatorAudit. +test("convex-test: writeOperatorAudit preserves generation and kind and sets the mutation time", async () => { + const t = gateApp(); + const [generationId] = await generationIds(t); + for (const kind of ["gate.close", "gate.resume"] as const) { + const record = { + kind, + scopeKey: "tenant:t", + reason: "repair", + generationId, + operator: " stated operator ", + }; + const result = await t.run(async (ctx) => ({ + at: Date.now(), + id: await writeOperatorAudit(ctx, record), + })); + expect(await t.run((ctx) => ctx.db.get(result.id))).toEqual({ + ...record, + _id: result.id, + _creationTime: expect.any(Number), + recordedAt: result.at, + }); + } +}); + +for (const kind of ["security", "business"] as const) { + for (const decision of ["applied", "businessFailure"] as const) { + // spec:operations.baseline-operations auditRecordSources; spec:command.command-pipeline auditWrite; spec:command.command-declaration typeAudit. + test(`convex-test: ${kind} audit of ${decision} takes every field from its specified source and uses subjectFrom`, async () => { + const t = gateApp(); + await grant(t); + const declaredSubject = { + contextId: "permission-context", + streamType: "permission-type", + streamId: "permission-d", + }; + const { decl, seen } = declaration({ + kind, + decision, + subjectFrom: (input) => ({ + ...declaredSubject, + streamId: `permission-${input.documentId}`, + }), + }); + const { response, at } = await t.run(async (ctx) => ({ + at: Date.now(), + response: await runPipeline(ctx, decl, call), + })); + expect(seen).toHaveLength(1); + expect( + await t.run((ctx) => ctx.db.query("auditRecords").collect()), + ).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: call.tenantId, + requestKey: call.requestKey, + actor, + operationId: seen[0]!.operationId, + causedBy: seen[0]!.causedBy, + commandType: "AuditedDocument", + kind, + decision, + subject: declaredSubject, + recordedAt: at, + }, + ]); + expect(seen[0]!.operationId).toBe(response.operationId); + expect(seen[0]!.causedBy).toEqual(explicitCause); + expect(response.versions[0]?.streamId).toBe("d"); + const rows = await t.run((ctx) => ctx.db.query("auditRecords").collect()); + expect(await t.run((ctx) => runPipeline(ctx, decl, call))).toMatchObject({ + replayed: true, + operationId: response.operationId, + }); + expect( + await t.run((ctx) => ctx.db.query("auditRecords").collect()), + ).toEqual(rows); + }); + } +} + +// spec:operations.baseline-operations auditRecordSources; spec:command.command-pipeline Contract "Step 7, gate and mint the operation". +test("convex-test: absent subjectFrom uses the first version, with the default command cause and no request key", async () => { + const t = gateApp(); + await grant(t); + const { decl, seen } = declaration(); + const unreceipted: PipelineCall = { + tenantId: "t", + namespace: "public", + actor, + input: call.input, + }; + const result = await t.run((ctx) => runPipeline(ctx, decl, unreceipted)); + expect(result.versions.map((version) => version.streamId)).toEqual([ + "d", + "second", + ]); + expect(await t.run((ctx) => ctx.db.query("auditRecords").collect())).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "t", + actor, + operationId: result.operationId, + causedBy: { kind: "command", commandType: "AuditedDocument" }, + commandType: "AuditedDocument", + kind: "business", + decision: "applied", + subject, + recordedAt: expect.any(Number), + }, + ]); + expect(seen[0]!.causedBy).toEqual({ + kind: "command", + commandType: "AuditedDocument", + }); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual([]); +}); + +// spec:operations.baseline-operations auditRecordSources. +test("convex-test: subjectFrom supplies audit even when the outcome has no version", async () => { + const t = gateApp(); + await grant(t); + const { decl } = declaration({ subjectFrom: () => subject, versions: false }); + await t.run((ctx) => runPipeline(ctx, decl, call)); + expect(await t.run((ctx) => ctx.db.query("auditRecords").collect())).toEqual([ + expect.objectContaining({ subject }), + ]); +}); + +// spec:operations.baseline-operations auditRecordSources; spec:command.command-pipeline auditWrite. +test("convex-test: an audited declaration without either subject source throws a plain error and rolls back", async () => { + const t = gateApp(); + await grant(t); + const { decl, seen } = declaration({ versions: false }); + const error = await caught(t.run((ctx) => runPipeline(ctx, decl, call))); + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(ConvexError); + expect(seen).toHaveLength(1); + expect(await depotRows(t)).toEqual({ events: [], streams: [] }); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual([]); + expect(await t.run((ctx) => ctx.db.query("auditRecords").collect())).toEqual( + [], + ); + expect( + await t.run((ctx) => + ctx.runQuery(components.depot.queries.document.get, { + tenantId: "t", + streamId: "d", + }), + ), + ).toBeNull(); +}); + +// spec:operations.baseline-operations fnWriteAudit, faultInjection; spec:command.command-pipeline auditWrite. +test("convex-test: a rejecting audit validator rolls back the receipt and context writes", async () => { + const t = gateApp("auditRecords"); + await grant(t); + const { decl, seen } = declaration(); + await expect(t.run((ctx) => runPipeline(ctx, decl, call))).rejects.toThrow(); + expect(seen).toHaveLength(1); + expect(await depotRows(t)).toEqual({ events: [], streams: [] }); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual([]); + expect(await t.run((ctx) => ctx.db.query("auditRecords").collect())).toEqual( + [], + ); + expect( + await t.run((ctx) => + ctx.runQuery(components.depot.queries.document.get, { + tenantId: "t", + streamId: "d", + }), + ), + ).toBeNull(); +}); + +// spec:command.command-declaration typeAudit; spec:command.command-pipeline auditWrite. +test("convex-test: a declaration without audit writes no audit record", async () => { + const t = gateApp(); + await grant(t); + const { decl } = declaration(); + delete decl.audit; + expect(await t.run((ctx) => runPipeline(ctx, decl, call))).toMatchObject({ + kind: "applied", + }); + expect(await t.run((ctx) => ctx.db.query("auditRecords").collect())).toEqual( + [], + ); +}); diff --git a/tests/simulator/gate-inspection.ts b/tests/simulator/gate-inspection.ts new file mode 100644 index 0000000..1cc0c5d --- /dev/null +++ b/tests/simulator/gate-inspection.ts @@ -0,0 +1,10 @@ +import { internalQueryGeneric } from "convex/server"; + +// Registered only in convex-test's depot module map, to inspect rollback at the journal boundary. +export const stored = internalQueryGeneric({ + args: {}, + handler: async (ctx) => ({ + events: await ctx.db.query("events").collect(), + streams: await ctx.db.query("streams").collect(), + }), +}); diff --git a/tests/simulator/gate-pipeline.test.ts b/tests/simulator/gate-pipeline.test.ts new file mode 100644 index 0000000..53baee0 --- /dev/null +++ b/tests/simulator/gate-pipeline.test.ts @@ -0,0 +1,183 @@ +import { v } from "convex/values"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { + api, + components, + internal, +} from "../../fixture/convex/_generated/api.js"; +import { permissions } from "../../fixture/convex/depotCommands.js"; +import { documentSummary } from "../../fixture/convex/summaries.js"; +import { + insertGrant, + runPipeline, + type CommandDeclaration, + type PipelineCall, +} from "../../src/command/index.js"; +import { + gateApp, + depotRows, + close, + resume, + refusal, + type GateApp, +} from "./gate-support.js"; + +beforeEach(() => { + vi.stubEnv("MAINTENANCE_MODE", undefined); + vi.spyOn(Date, "now").mockReturnValue(1_000_000); +}); +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); +const issuer = "https://gate.test"; +async function caller(t: GateApp) { + await t.run((ctx) => + insertGrant(ctx, { + tenantId: "t", + principalKind: "human", + principalId: `${issuer}|caller`, + permission: permissions.documents, + grantedBy: "admin", + }), + ); + return t.withIdentity({ issuer, subject: "caller" }); +} +const call = { + tenantId: "t", + requestKey: "same-key", + input: { documentId: "d", title: "Document" }, +}; +async function stored(t: GateApp) { + const parent = await t.run(async (ctx) => ({ + receipts: await ctx.db.query("receipts").collect(), + rows: await ctx.db.query("documentSummaries").collect(), + audits: await ctx.db.query("auditRecords").collect(), + })); + const depot = await t.run(async (ctx) => + ctx.runQuery(components.depot.queries.document.get, { + tenantId: "t", + streamId: "d", + }), + ); + return { ...parent, depot, ...(await depotRows(t)) }; +} + +// spec:application.write-pause errorCodeWritePaused; spec:command.command-pipeline gateRead, auditWrite. +test("convex-test: a paused audited command stores no receipt, event, state, read-model row or audit and its key applies after resume", async () => { + const t = gateApp(); + await caller(t); + await t.mutation(internal.readModels.activate, { + readModel: "documentSummary", + startedBy: { kind: "operator", id: "admin" }, + }); + const declaration: CommandDeclaration< + { documentId: string; title: string }, + null + > = { + name: "CreateAuditedDocument", + contractVersion: 1, + input: v.object({ documentId: v.string(), title: v.string() }), + output: v.null(), + permission: { permission: permissions.documents }, + writes: [{ contextId: "depot", streamType: "document" }], + readModels: [ + { + readModel: documentSummary, + source: { contextId: "depot", streamType: "document" }, + }, + ], + rejections: ["titleRequired"], + audit: { kind: "business" }, + executor: async (ctx, { tenantId, actor, operation, input }) => { + const outcome = await ctx.runMutation( + components.depot.operations.createDocuments, + { tenantId, actor, operation, input: { documents: [input] } }, + ); + return { ...outcome, result: null }; + }, + }; + const pipelineCall: PipelineCall = { + ...call, + namespace: "public", + actor: { kind: "human", id: `${issuer}|caller` }, + }; + const before = await stored(t); + await close(t); + const execute = vi.fn(declaration.executor); + declaration.executor = execute; + const run = () => t.run((ctx) => runPipeline(ctx, declaration, pipelineCall)); + const mint = vi.spyOn(crypto, "randomUUID"); + await refusal(run(), "tenant:t", "repair"); + expect(mint).not.toHaveBeenCalled(); + expect(await stored(t)).toEqual(before); + expect(before.receipts).toEqual([]); + expect(before.rows).toEqual([]); + expect(before.audits).toEqual([]); + expect(execute).not.toHaveBeenCalled(); + expect(before.depot).toBeNull(); + expect(before.events).toEqual([]); + expect(before.streams).toEqual([]); + mint.mockRestore(); + await resume(t); + expect(await run()).toMatchObject({ kind: "applied", replayed: false }); + const after = await stored(t); + expect(after.receipts).toHaveLength(1); + expect(after.rows).toHaveLength(1); + expect(after.audits).toHaveLength(1); + expect(after.events).toHaveLength(1); + expect(after.streams).toHaveLength(1); +}); + +// spec:command.command-pipeline Contract "Step 5, receipt lookup", "Step 6, admit", gateRead; spec:command.tenancy-and-authority rule "Authorization comes before execution". +test("convex-test: a duplicate in a closed tenant returns its receipt before admission and the gate", async () => { + const t = gateApp(); + const user = await caller(t); + const first = await user.mutation(api.depotCommands.createDocument, call); + await close(t); + await t.mutation(internal.switches.set, { + tenantId: "t", + commandType: "CreateDocument", + name: "capacity", + on: true, + }); + const before = await stored(t); + expect( + await user.mutation(api.depotCommands.createDocument, call), + ).toMatchObject({ replayed: true, operationId: first.operationId }); + expect(await stored(t)).toEqual(before); + await t.mutation(internal.grants.revoke, { + tenantId: "t", + principalKind: "human", + principalId: `${issuer}|caller`, + permission: permissions.documents, + }); + await expect( + user.mutation(api.depotCommands.createDocument, call), + ).rejects.toMatchObject({ data: { kind: "rejection", code: "forbidden" } }); +}); + +// spec:command.command-pipeline Contract "Step 6, admit", gateRead. +test("convex-test: admission refuses new intent before the gate and no operation is minted", async () => { + const t = gateApp(); + const user = await caller(t); + await close(t); + await t.mutation(internal.switches.set, { + tenantId: "t", + commandType: "CreateDocument", + name: "capacity", + on: true, + }); + const mint = vi.spyOn(crypto, "randomUUID"); + await expect( + user.mutation(api.depotCommands.createDocument, call), + ).rejects.toMatchObject({ + data: { + kind: "transient", + code: "capacity", + message: "CreateDocument is not admitted now", + }, + }); + expect(mint).not.toHaveBeenCalled(); + expect((await stored(t)).receipts).toEqual([]); +}); diff --git a/tests/simulator/gate-support.ts b/tests/simulator/gate-support.ts new file mode 100644 index 0000000..c5289a9 --- /dev/null +++ b/tests/simulator/gate-support.ts @@ -0,0 +1,122 @@ +import { convexTest } from "convex-test"; +import { + componentsGeneric, + defineSchema, + defineTable, + type FunctionReference, +} from "convex/server"; +import { ConvexError, v, type Value } from "convex/values"; +import { expect } from "vitest"; +import { internal } from "../../fixture/convex/_generated/api.js"; +import annexSchema from "../../fixture/convex/annex/schema.js"; +import depotSchema from "../../fixture/convex/depot/schema.js"; +import schema from "../../fixture/convex/schema.js"; + +export function gateApp(refuseAudit?: "operatorAudit" | "auditRecords") { + // Only the validator differs. Production exports and handlers still run unchanged. + const selected = + refuseAudit === undefined + ? schema + : (defineSchema({ + ...schema.tables, + [refuseAudit]: defineTable({ + impossible: v.literal("refuse every audit record"), + }), + }) as unknown as typeof schema); + const t = convexTest( + selected, + import.meta.glob("../../fixture/convex/**/*.ts"), + ); + t.registerComponent( + "annex", + annexSchema, + import.meta.glob("../../fixture/convex/annex/**/*.ts"), + ); + t.registerComponent("depot", depotSchema, { + ...import.meta.glob("../../fixture/convex/depot/**/*.ts"), + "../../fixture/convex/depot/gateInspection.ts": () => + import("./gate-inspection.js"), + }); + return t; +} +export type GateApp = ReturnType; +export const operator = " stated operator "; +export const close = ( + t: GateApp, + scopeKey = "tenant:t", + reason = "repair", + statedOperator = operator, +) => + t.mutation(internal.gate.closeGate, { + scopeKey, + reason, + operator: statedOperator, + }); +export const resume = ( + t: GateApp, + scopeKey = "tenant:t", + statedOperator = operator, +) => + t.mutation(internal.gate.resumeGate, { scopeKey, operator: statedOperator }); +export const gateRows = (t: GateApp) => + t.run((ctx) => ctx.db.query("maintenanceGates").collect()); +export const operatorRows = (t: GateApp) => + t.run((ctx) => ctx.db.query("operatorAudit").collect()); +export async function caught(promise: Promise): Promise { + return promise.then( + () => { + throw new Error("Expected the call to fail"); + }, + (error: unknown) => error, + ); +} +export async function plainFailure(promise: Promise, message: string) { + const error = await caught(promise); + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(ConvexError); + expect((error as Error).message).toBe(message); +} +export async function refusal( + promise: Promise, + scopeKey: string, + reason: string, +) { + const error = await caught(promise); + expect(error).toBeInstanceOf(ConvexError); + expect((error as ConvexError).data).toStrictEqual({ + kind: "transient", + code: "writePaused", + message: `write paused for ${scopeKey}: ${reason}`, + }); +} +export async function generationIds(t: GateApp) { + await t.mutation(internal.readModels.activate, { + readModel: "documentSummary", + startedBy: { kind: "operator", id: "setup" }, + }); + await t.mutation(internal.readModels.activate, { + readModel: "documentTitle", + startedBy: { kind: "operator", id: "setup" }, + }); + const rows = await t.run((ctx) => ctx.db.query("generations").collect()); + const [first, second] = rows; + if (!first || !second) throw new Error("Two generation rows are required"); + return [first._id, second._id] as const; +} + +export const inspection = ( + componentsGeneric() as unknown as { + depot: { + gateInspection: { + stored: FunctionReference< + "query", + "internal", + Record, + { events: Value[]; streams: Value[] } + >; + }; + }; + } +).depot.gateInspection.stored; +export const depotRows = (t: GateApp) => + t.run((ctx) => ctx.runQuery(inspection, {})); diff --git a/tests/simulator/gate.test.ts b/tests/simulator/gate.test.ts new file mode 100644 index 0000000..a49cb4a --- /dev/null +++ b/tests/simulator/gate.test.ts @@ -0,0 +1,577 @@ +import type { FunctionReturnType } from "convex/server"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { internal } from "../../fixture/convex/_generated/api.js"; +import { + assertWritable, + gateAllows, + closeScope, + resumeScope, +} from "../../src/gate/index.js"; +import { gateHandler, gateQueryHandler } from "../pure/gate-handlers.js"; +import { + close, + resume, + gateApp, + gateRows, + operatorRows, + plainFailure, + refusal, + generationIds, + operator, +} from "./gate-support.js"; + +beforeEach(() => { + vi.stubEnv("MAINTENANCE_MODE", undefined); + vi.spyOn(Date, "now").mockReturnValue(1_000_000); +}); +afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); +}); + +// spec:application.write-pause fnGateAllows, restoreDoor, errorCodeWritePaused. +test("convex-test: restore refuses every scope and generation without touching the database", async () => { + const t = gateApp(); + const [generation] = await generationIds(t); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + await t.run(async (ctx) => { + const access = vi.fn(() => { + throw new Error("Database accessed under restore"); + }); + const db = new Proxy(ctx.db, { get: access }); + for (const scopes of [[], ["all"], ["tenant:t"]]) { + expect(await gateAllows({ db }, scopes, generation)).toEqual({ + allowed: false, + scopeKey: "all", + reason: "restore", + }); + await refusal(assertWritable({ db }, scopes), "all", "restore"); + } + expect(access).not.toHaveBeenCalled(); + }); +}); + +// spec:application.write-pause fnGateAllows, restoreDoor, indexGatesByKeyUse. +test.each(["Restore", "restore ", "", undefined])( + "convex-test: environment value %j defers to exactly one gate read", + async (value) => { + const t = gateApp(); + await close(t); + vi.stubEnv("MAINTENANCE_MODE", value); + await t.run(async (ctx) => { + const query = vi.spyOn(ctx.db, "query"); + expect(await gateAllows(ctx, ["all", "tenant:t"])).toEqual({ + allowed: false, + scopeKey: "tenant:t", + reason: "repair", + }); + expect(query.mock.calls).toEqual([["maintenanceGates"]]); + query.mockRestore(); + expect(await gateAllows(ctx, ["all", "tenant:other"])).toEqual({ + allowed: true, + }); + }); + }, +); + +// spec:application.write-pause Contract "A deployment with no gate document is open", fnGateAllows. +test("convex-test: an absent gate is open and neither read helper writes", async () => { + const t = gateApp(); + await t.run(async (ctx) => { + const writes = [ + vi.spyOn(ctx.db, "insert"), + vi.spyOn(ctx.db, "patch"), + vi.spyOn(ctx.db, "replace"), + vi.spyOn(ctx.db, "delete"), + ]; + expect(await gateAllows(ctx, ["all", "tenant:t"])).toEqual({ + allowed: true, + }); + await assertWritable(ctx, ["all", "tenant:t"]); + for (const write of writes) expect(write).not.toHaveBeenCalled(); + }); + expect(await gateRows(t)).toEqual([]); +}); + +// spec:application.write-pause fnGateAllows, tableMaintenanceGates. +test("convex-test: the first matching entry wins in closed array order", async () => { + const t = gateApp(); + await close(t, "tenant:other", "unmatched"); + await close(t, "tenant:t", "tenant first"); + await close(t, "all", "deployment last"); + expect(await t.run((ctx) => gateAllows(ctx, ["all", "tenant:t"]))).toEqual({ + allowed: false, + scopeKey: "tenant:t", + reason: "tenant first", + }); + await resume(t); + expect(await t.run((ctx) => gateAllows(ctx, ["tenant:t", "all"]))).toEqual({ + allowed: false, + scopeKey: "all", + reason: "deployment last", + }); +}); + +// spec:application.write-pause fnGateAllows, Contract "The rebuild a scope was closed for". +test("convex-test: only an explicitly matching generation exempts an entry", async () => { + const t = gateApp(); + const [first, second] = await generationIds(t); + await close(t); + const read = (id?: typeof first) => + t.run((ctx) => gateAllows(ctx, ["all", "tenant:t"], id)); + const closed = { allowed: false, scopeKey: "tenant:t", reason: "repair" }; + expect(await read()).toEqual(closed); + expect(await read(first)).toEqual(closed); + await t.run(async (ctx) => { + const row = await ctx.db.query("maintenanceGates").unique(); + if (!row) throw new Error("Gate missing"); + await ctx.db.patch(row._id, { + closed: row.closed.map((entry) => ({ ...entry, generationId: first })), + }); + }); + expect(await read()).toEqual(closed); + expect(await read(second)).toEqual(closed); + expect(await read(first)).toEqual({ allowed: true }); + await close(t, "all", "another entry"); + expect(await read(first)).toEqual({ + allowed: false, + scopeKey: "all", + reason: "another entry", + }); +}); + +// spec:application.write-pause fnCloseScope, fnResumeScope, validatorClosedEntry; spec:operations.baseline-operations tableOperatorAudit. +test("convex-test: close and resume record exactly one audit each with unchanged operators", async () => { + const t = gateApp(); + const before = Date.now(); + expect(await close(t)).toBeNull(); + const [row] = await gateRows(t); + expect(row).toEqual({ + _id: expect.any(String), + _creationTime: expect.any(Number), + key: "gates", + closed: [ + { + scopeKey: "tenant:t", + reason: "repair", + changedAt: expect.any(Number), + changedBy: operator, + }, + ], + updatedAt: expect.any(Number), + }); + expect(row!.updatedAt).toBe(row!.closed[0]!.changedAt); + expect(row!.updatedAt).toBeGreaterThanOrEqual(before); + expect(row!.updatedAt).toBeLessThanOrEqual(Date.now()); + expect(await operatorRows(t)).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + kind: "gate.close", + scopeKey: "tenant:t", + reason: "repair", + operator, + recordedAt: row!.updatedAt, + }, + ]); + const frozen = await gateRows(t); + await plainFailure(close(t), "The scope tenant:t is already closed"); + expect(await gateRows(t)).toEqual(frozen); + expect(await operatorRows(t)).toHaveLength(1); + expect(await resume(t, "tenant:t", "other operator")).toBeNull(); + const [opened] = await gateRows(t); + expect(opened?.closed).toEqual([]); + expect(await operatorRows(t)).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + kind: "gate.close", + scopeKey: "tenant:t", + reason: "repair", + operator, + recordedAt: row!.updatedAt, + }, + { + _id: expect.any(String), + _creationTime: expect.any(Number), + kind: "gate.resume", + scopeKey: "tenant:t", + reason: "repair", + operator: "other operator", + recordedAt: opened!.updatedAt, + }, + ]); +}); + +// spec:application.write-pause fnResumeScope. +test("convex-test: resuming an absent or already open scope changes nothing", async () => { + const t = gateApp(); + await plainFailure(resume(t), "The scope tenant:t is not closed"); + expect(await gateRows(t)).toEqual([]); + expect(await operatorRows(t)).toEqual([]); + await close(t); + await resume(t); + const before = await gateRows(t); + await plainFailure(resume(t), "The scope tenant:t is not closed"); + expect(await gateRows(t)).toEqual(before); + expect(await operatorRows(t)).toHaveLength(2); +}); + +// spec:application.write-pause fnCloseScope, limitClosedScopes. +test("convex-test: the sixteenth scope closes and the seventeenth is refused", async () => { + const t = gateApp(); + for (let i = 0; i < 16; i++) await close(t, `tenant:${i}`); + const before = await gateRows(t); + expect(before[0]?.closed.map((entry) => entry.scopeKey)).toEqual( + Array.from({ length: 16 }, (_, i) => `tenant:${i}`), + ); + await plainFailure( + close(t, "tenant:16"), + "The gate holds 16 closed scopes, its limit", + ); + expect(await gateRows(t)).toEqual(before); + expect(await operatorRows(t)).toHaveLength(16); + await resume(t, "tenant:0"); + await close(t, "tenant:16"); + expect((await gateRows(t))[0]?.closed).toHaveLength(16); +}); + +// spec:application.write-pause fnCloseGate, limitGateReason. +test.each(["a", " ", "a".repeat(256), "é".repeat(128), "😀".repeat(64)])( + "convex-test: a reason within the byte bound is stored unchanged, %j", + async (reason) => { + const t = gateApp(); + await close(t, "tenant:t", reason); + expect((await gateRows(t))[0]?.closed[0]?.reason).toBe(reason); + expect((await operatorRows(t))[0]?.reason).toBe(reason); + }, +); + +// spec:application.write-pause fnCloseGate, limitGateReason. +test.each(["", "a".repeat(257), "é".repeat(128) + "a", "😀".repeat(64) + "é"])( + "convex-test: a reason outside the byte bound changes nothing, %j", + async (reason) => { + const t = gateApp(); + await plainFailure( + close(t, "tenant:t", reason), + "The reason must be between 1 and 256 bytes", + ); + expect(await gateRows(t)).toEqual([]); + expect(await operatorRows(t)).toEqual([]); + }, +); + +for (const entry of ["closeGate", "resumeGate"] as const) { + // spec:application.write-pause fnCloseGate, fnResumeGate. + test(`convex-test: ${entry} refuses a source scope with its own name`, async () => { + const t = gateApp(); + const scopeKey = "source:depot:document"; + await plainFailure( + entry === "closeGate" ? close(t, scopeKey) : resume(t, scopeKey), + `${entry} takes the scope all or a tenant scope, and ${scopeKey} is neither`, + ); + expect(await gateRows(t)).toEqual([]); + expect(await operatorRows(t)).toEqual([]); + }); + + // spec:command.actor-and-scope fnAssertOperator, Contract "An operator entry that changes anything". + test.each(["", " \t", "a".repeat(513), "é".repeat(257)])( + `convex-test: ${entry} refuses operator %j before any context access`, + async (stated) => { + const t = gateApp(); + await t.run(async (ctx) => { + const access = vi.fn(() => { + throw new Error("Context accessed"); + }); + const guarded = new Proxy(ctx, { get: access }); + const message = + stated.trim() === "" + ? "An operator entry needs a stated operator" + : `The stated operator is ${new TextEncoder().encode(stated).length} bytes, above the limit of 512`; + await plainFailure( + gateHandler(entry)._handler(guarded, { + scopeKey: "source:depot:document", + reason: "", + operator: stated, + }), + message, + ); + expect(access).not.toHaveBeenCalled(); + }); + expect(await gateRows(t)).toEqual([]); + expect(await operatorRows(t)).toEqual([]); + }, + ); +} + +// spec:command.actor-and-scope Contract "An operator entry is an internalMutation", fnAssertOperator; spec:application.write-pause fnCloseGate. +test.each(["a".repeat(512), "é".repeat(256)])( + "convex-test: bounded operators change a tenant with no grant and no auth read, %j", + async (stated) => { + const t = gateApp(); + await t.run(async (ctx) => { + const query = vi.spyOn(ctx.db, "query"); + const auth = vi.spyOn(ctx.auth, "getUserIdentity"); + for (const entry of ["closeGate", "resumeGate"] as const) + await gateHandler(entry)._handler(ctx, { + scopeKey: "tenant:source:depot:document", + reason: "repair", + operator: stated, + }); + expect( + query.mock.calls.every(([table]) => table === "maintenanceGates"), + ).toBe(true); + expect(auth).not.toHaveBeenCalled(); + }); + expect((await operatorRows(t)).map((row) => row.operator)).toEqual([ + stated, + stated, + ]); + expect(await t.run((ctx) => ctx.db.query("tenants").collect())).toEqual([]); + }, +); + +// spec:application.write-pause fnCloseScope, fnResumeScope, Contract "Gate changes are treated as mandatory audit". +test("convex-test: close and resume in one mutation leave two audit records and an open gate", async () => { + const t = gateApp(); + await t.run(async (ctx) => { + await closeScope(ctx, "all", "repair", "closer"); + await resumeScope(ctx, "all", "resumer"); + }); + expect((await gateRows(t))[0]?.closed).toEqual([]); + expect( + (await operatorRows(t)).map( + ({ kind, operator: stated, reason, scopeKey }) => ({ + kind, + operator: stated, + reason, + scopeKey, + }), + ), + ).toEqual([ + { + kind: "gate.close", + operator: "closer", + reason: "repair", + scopeKey: "all", + }, + { + kind: "gate.resume", + operator: "resumer", + reason: "repair", + scopeKey: "all", + }, + ]); +}); + +// spec:application.write-pause fnResumeScope, fnResumeGate; spec:operations.baseline-operations fnWriteOperatorAudit. +test("convex-test: resumeGate copies the removed entry's generation and reason to audit", async () => { + const t = gateApp(); + const [generationId] = await generationIds(t); + await t.run((ctx) => + ctx.db.insert("maintenanceGates", { + key: "gates", + closed: [ + { + scopeKey: "tenant:t", + reason: "restored entry", + generationId, + changedBy: "closer", + changedAt: 1, + }, + ], + updatedAt: 1, + }), + ); + await resume(t); + expect((await gateRows(t))[0]?.closed).toEqual([]); + expect(await operatorRows(t)).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + kind: "gate.resume", + scopeKey: "tenant:t", + reason: "restored entry", + generationId, + operator, + recordedAt: expect.any(Number), + }, + ]); +}); + +// spec:application.write-pause Contract "Gate changes are treated as mandatory audit"; spec:operations.baseline-operations faultInjection. +test.each(["absent", "existing", "resume"])( + "convex-test: a refused operator audit leaves the %s gate unchanged", + async (state) => { + const t = gateApp("operatorAudit"); + if (state !== "absent") + await t.run((ctx) => + ctx.db.insert("maintenanceGates", { + key: "gates", + closed: [ + { + scopeKey: "tenant:t", + reason: "original", + changedAt: 1, + changedBy: "original operator", + }, + ], + updatedAt: 1, + }), + ); + const before = await gateRows(t); + await expect( + state === "resume" ? resume(t) : close(t, "all"), + ).rejects.toThrow(); + expect(await gateRows(t)).toEqual(before); + expect(await operatorRows(t)).toEqual([]); + }, +); + +// spec:application.write-pause fnGetGate. +test("convex-test: getGate reports restore separately and preserves document entries", async () => { + const t = gateApp(); + expect(await t.query(internal.gate.getGate, {})).toEqual({ + restore: false, + closed: [], + }); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + expect(await t.query(internal.gate.getGate, {})).toEqual({ + restore: true, + closed: [], + }); + await close(t); + expect(await t.query(internal.gate.getGate, {})).toEqual({ + restore: true, + closed: (await gateRows(t))[0]!.closed, + }); + vi.stubEnv("MAINTENANCE_MODE", "Restore"); + expect(await t.query(internal.gate.getGate, {})).toEqual({ + restore: false, + closed: (await gateRows(t))[0]!.closed, + }); + await resume(t); +}); + +// spec:application.write-pause fnGetGateAudit; spec:operations.baseline-operations limitOperatorQuery, tableOperatorAudit. +test("convex-test: getGateAudit pages one scope newest first with at most 100 records", async () => { + const t = gateApp(); + await t.run(async (ctx) => { + for (let i = 0; i < 205; i++) + await ctx.db.insert("operatorAudit", { + kind: i % 2 === 0 ? "gate.close" : "gate.resume", + scopeKey: "tenant:t", + reason: `reason-${i}`, + operator: "op", + recordedAt: i, + }); + await ctx.db.insert("operatorAudit", { + kind: "gate.close", + scopeKey: "tenant:other", + reason: "other", + operator: "op", + recordedAt: 999, + }); + }); + const expected = (await operatorRows(t)) + .filter((row) => row.scopeKey === "tenant:t") + .sort((a, b) => b.recordedAt - a.recordedAt); + const rows = []; + let cursor: string | null = null; + let done = false; + for (let page = 0; page < 4 && !done; page++) { + const result: FunctionReturnType = + await t.query(internal.gate.getGateAudit, { + scopeKey: "tenant:t", + paginationOpts: { numItems: 1000, cursor }, + }); + expect(result.page.length).toBeGreaterThan(0); + expect(result.page.length).toBeLessThanOrEqual(100); + rows.push(...result.page); + cursor = result.continueCursor; + done = result.isDone; + } + expect(done).toBe(true); + expect(rows).toEqual(expected); + const small = await t.query(internal.gate.getGateAudit, { + scopeKey: "tenant:t", + paginationOpts: { numItems: 1, cursor: null }, + }); + expect(small.page).toEqual(expected.slice(0, 1)); +}); + +// spec:application.write-pause fnCloseScope; spec:operations.baseline-operations tableOperatorAudit, fnWriteOperatorAudit. +test("convex-test: closeScope copies an existing generation into the entry and its audit", async () => { + const t = gateApp(); + const [generationId] = await generationIds(t); + await t.run((ctx) => + closeScope(ctx, "tenant:t", "repair", operator, generationId), + ); + const [gate] = await gateRows(t); + expect(gate?.closed).toEqual([ + { + scopeKey: "tenant:t", + reason: "repair", + generationId, + changedAt: 1_000_000, + changedBy: operator, + }, + ]); + expect(await operatorRows(t)).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + kind: "gate.close", + scopeKey: "tenant:t", + reason: "repair", + generationId, + operator, + recordedAt: 1_000_000, + }, + ]); +}); + +// spec:command.actor-and-scope Contract "An operator entry is an internalMutation", "An operator entry that changes anything"; spec:application.write-pause fnGetGate, fnGetGateAudit. +test("convex-test: the reading entries require no operator and read neither grants nor auth", async () => { + const t = gateApp(); + await close(t); + await t.run(async (ctx) => { + const query = vi.spyOn(ctx.db, "query"); + const auth = vi.spyOn(ctx.auth, "getUserIdentity"); + const insert = vi.spyOn(ctx.db, "insert"); + const patch = vi.spyOn(ctx.db, "patch"); + const remove = vi.spyOn(ctx.db, "delete"); + const replace = vi.spyOn(ctx.db, "replace"); + await gateQueryHandler("getGate")._handler(ctx, {}); + await gateQueryHandler("getGateAudit")._handler(ctx, { + scopeKey: "tenant:t", + paginationOpts: { numItems: 100, cursor: null }, + }); + expect(query.mock.calls).toEqual([["maintenanceGates"], ["operatorAudit"]]); + for (const access of [auth, insert, patch, remove, replace]) + expect(access).not.toHaveBeenCalled(); + }); +}); + +// spec:application.write-pause typeScopeKey, fnCloseGate, fnGateAllows. +test("convex-test: colons inside a tenant ID remain part of that tenant's closed scope", async () => { + const t = gateApp(); + await close(t, "tenant:source:depot:document"); + expect((await gateRows(t))[0]?.closed[0]?.scopeKey).toBe( + "tenant:source:depot:document", + ); + expect( + await t.run((ctx) => + gateAllows(ctx, ["all", "tenant:source:depot:document"]), + ), + ).toEqual({ + allowed: false, + scopeKey: "tenant:source:depot:document", + reason: "repair", + }); + expect( + await t.run((ctx) => gateAllows(ctx, ["all", "tenant:source"])), + ).toEqual({ allowed: true }); + await resume(t, "tenant:source:depot:document"); + expect((await gateRows(t))[0]?.closed).toEqual([]); +}); diff --git a/tests/simulator/tenants.test.ts b/tests/simulator/tenants.test.ts new file mode 100644 index 0000000..2dc0ab9 --- /dev/null +++ b/tests/simulator/tenants.test.ts @@ -0,0 +1,80 @@ +import { convexTest } from "convex-test"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import schema from "../../fixture/convex/schema.js"; +import { + insertGrant, + nextTenant, + revokeGrant, +} from "../../src/command/index.js"; +beforeEach(() => vi.spyOn(Date, "now").mockReturnValue(1_000_000)); +afterEach(() => vi.restoreAllMocks()); +const app = () => + convexTest(schema, import.meta.glob("../../fixture/convex/**/*.ts")); +const grant = (tenantId: string, permission = "documents") => ({ + tenantId, + principalKind: "service" as const, + principalId: "caller", + permission, + grantedBy: "admin", +}); + +// spec:command.actor-and-scope fnInsertGrant, tableTenants, Contract "The tenant list is the tenants table". +test("convex-test: the first grant creates one tenant row, later grants and all revocations retain it", async () => { + const t = app(); + const at = await t.run(async (ctx) => { + const now = Date.now(); + await insertGrant(ctx, grant("t")); + return now; + }); + const first = await t.run((ctx) => ctx.db.query("tenants").collect()); + expect(first).toEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "t", + createdAt: at, + }, + ]); + expect(await t.run((ctx) => ctx.db.query("grants").collect())).toEqual([ + expect.objectContaining({ tenantId: "t", grantedAt: at }), + ]); + await t.run((ctx) => insertGrant(ctx, grant("t", "stock"))); + expect(await t.run((ctx) => ctx.db.query("tenants").collect())).toEqual( + first, + ); + await t.run(async (ctx) => { + expect(await revokeGrant(ctx, grant("t"))).toBe(1); + expect(await revokeGrant(ctx, grant("t", "stock"))).toBe(1); + }); + expect(await t.run((ctx) => ctx.db.query("grants").collect())).toEqual([]); + expect(await t.run((ctx) => ctx.db.query("tenants").collect())).toEqual( + first, + ); +}); + +// spec:command.actor-and-scope fnNextTenant, Contract "A pass over the whole deployment". +test("convex-test: nextTenant reads one tenant in ascending ID order, excludes the cursor and ends at null", async () => { + const t = app(); + expect(await t.run((ctx) => nextTenant(ctx, null))).toBeNull(); + for (const id of ["z", "b", "a", "tenant:source:depot"]) + await t.run((ctx) => insertGrant(ctx, grant(id))); + for (const [after, expected] of [ + [null, "a"], + ["a", "b"], + ["b", "tenant:source:depot"], + ["tenant:source:depot", "z"], + ["z", null], + ["az", "b"], + ] as const) { + await t.run(async (ctx) => { + const query = vi.spyOn(ctx.db, "query"); + expect(await nextTenant(ctx, after)).toBe(expected); + expect(query.mock.calls).toEqual([["tenants"]]); + query.mockRestore(); + }); + } + await t.run((ctx) => insertGrant(ctx, grant("aa"))); + expect(await t.run((ctx) => nextTenant(ctx, "b"))).toBe( + "tenant:source:depot", + ); +}); diff --git a/tests/types/gate-audit.test-d.ts b/tests/types/gate-audit.test-d.ts new file mode 100644 index 0000000..b8bd1b6 --- /dev/null +++ b/tests/types/gate-audit.test-d.ts @@ -0,0 +1,173 @@ +import type { + DataModelFromSchemaDefinition, + DocumentByName, + FunctionArgs, + FunctionReturnType, + FunctionReference, + PaginationOptions, + PaginationResult, + SchemaDefinition, +} from "convex/server"; +import type { GenericId, Infer } from "convex/values"; +import { expectTypeOf, test } from "vitest"; +import { internal as fixtureInternal } from "../../fixture/convex/_generated/api.js"; +import { internal as exampleInternal } from "../../example/convex/_generated/api.js"; +import { + auditTables, + type AuditRecordInput, + type OperatorAuditInput, +} from "../../src/audit/index.js"; +import { + operatorValidator, + type Actor, + type SubjectRef, + type CommandDeclaration, +} from "../../src/command/index.js"; +import type { CausedBy } from "../../src/context/index.js"; + +type AuditModel = DataModelFromSchemaDefinition< + SchemaDefinition +>; +type AuditDoc = DocumentByName; +type OperatorDoc = DocumentByName; +type ClosedEntry = { + scopeKey: string; + reason: string; + generationId?: GenericId<"generations">; + changedAt: number; + changedBy: string; +}; + +// spec:operations.baseline-operations typeAuditRecordInput, tableAuditRecords. +test("compiled: AuditRecordInput is the stored data without recordedAt", () => { + expectTypeOf().not.toBeAny(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + Omit + >(); + expectTypeOf().toEqualTypeOf<{ + tenantId: string; + operationId: string; + requestKey?: string; + commandType: string; + actor: Actor; + subject: SubjectRef; + kind: "security" | "business"; + decision: "applied" | "businessFailure"; + causedBy: CausedBy; + }>(); +}); + +// spec:operations.baseline-operations tableOperatorAudit, fnWriteOperatorAudit; spec:command.actor-and-scope validatorOperator. +test("compiled: operator audit uses kind and the operator validator is a string", () => { + expectTypeOf().not.toBeAny(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + Omit + >(); + expectTypeOf().toEqualTypeOf<{ + kind: "gate.close" | "gate.resume"; + scopeKey: string; + reason: string; + generationId?: GenericId<"generations">; + operator: string; + }>(); + expectTypeOf>().toEqualTypeOf(); +}); + +// spec:command.command-declaration typeAudit. +test("compiled: a declaration opts into security or business audit", () => { + expectTypeOf["audit"]>().toEqualTypeOf< + { kind: "security" | "business" } | undefined + >(); +}); + +// spec:application.write-pause fnCloseGate, fnResumeGate, fnGetGate, fnGetGateAudit. +test("compiled: the fixture exposes the pinned internal gate entries", () => { + const gate = fixtureInternal.gate; + expectTypeOf(gate).not.toBeAny(); + expectTypeOf(gate.closeGate).toExtend< + FunctionReference<"mutation", "internal"> + >(); + expectTypeOf(gate.resumeGate).toExtend< + FunctionReference<"mutation", "internal"> + >(); + expectTypeOf(gate.getGate).toExtend>(); + expectTypeOf(gate.getGateAudit).toExtend< + FunctionReference<"query", "internal"> + >(); + expectTypeOf>().toEqualTypeOf<{ + scopeKey: string; + reason: string; + operator: string; + }>(); + expectTypeOf>().toEqualTypeOf<{ + scopeKey: string; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + Record + >(); + expectTypeOf>().toEqualTypeOf<{ + restore: boolean; + closed: ClosedEntry[]; + }>(); + expectTypeOf>().toEqualTypeOf<{ + scopeKey: string; + paginationOpts: PaginationOptions; + }>(); + expectTypeOf>().toEqualTypeOf< + PaginationResult + >(); +}); + +// spec:application.write-pause fnCloseGate, fnResumeGate, fnGetGate, fnGetGateAudit. +test("compiled: the example exposes the pinned internal gate entries", () => { + const gate = exampleInternal.gate; + expectTypeOf(gate).not.toBeAny(); + expectTypeOf(gate.closeGate).toExtend< + FunctionReference<"mutation", "internal"> + >(); + expectTypeOf(gate.resumeGate).toExtend< + FunctionReference<"mutation", "internal"> + >(); + expectTypeOf(gate.getGate).toExtend>(); + expectTypeOf(gate.getGateAudit).toExtend< + FunctionReference<"query", "internal"> + >(); + expectTypeOf>().toEqualTypeOf<{ + scopeKey: string; + reason: string; + operator: string; + }>(); + expectTypeOf>().toEqualTypeOf<{ + scopeKey: string; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + Record + >(); + expectTypeOf>().toEqualTypeOf<{ + restore: boolean; + closed: ClosedEntry[]; + }>(); + expectTypeOf>().toEqualTypeOf<{ + scopeKey: string; + paginationOpts: PaginationOptions; + }>(); + expectTypeOf>().toEqualTypeOf< + PaginationResult + >(); +}); From 04e7e01437a1a7e4ffd9abb6fa6cb39f2537203e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:56:49 +0200 Subject: [PATCH 042/155] test: name the gate's suites after the write pause Co-Authored-By: Claude Opus 5.5 --- tests/native/{gate.test.ts => write-pause.test.ts} | 0 tests/pure/{gate.test.ts => write-pause.test.ts} | 0 tests/simulator/{gate.test.ts => write-pause.test.ts} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename tests/native/{gate.test.ts => write-pause.test.ts} (100%) rename tests/pure/{gate.test.ts => write-pause.test.ts} (100%) rename tests/simulator/{gate.test.ts => write-pause.test.ts} (100%) diff --git a/tests/native/gate.test.ts b/tests/native/write-pause.test.ts similarity index 100% rename from tests/native/gate.test.ts rename to tests/native/write-pause.test.ts diff --git a/tests/pure/gate.test.ts b/tests/pure/write-pause.test.ts similarity index 100% rename from tests/pure/gate.test.ts rename to tests/pure/write-pause.test.ts diff --git a/tests/simulator/gate.test.ts b/tests/simulator/write-pause.test.ts similarity index 100% rename from tests/simulator/gate.test.ts rename to tests/simulator/write-pause.test.ts From 012f37d0cd42bf5d57658e2ef57cf0690dd8e9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:57:24 +0200 Subject: [PATCH 043/155] design: what the scheduler and restore probe showed, on the restore, obligation, do-nothing and decision Specs Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- design/specs/application/restore.sdp.md | 6 +++--- .../decisions/d13-deferred-work-is-an-obligation.sdp.md | 4 ++-- .../decisions/d19-operations-travel-with-capability.sdp.md | 2 +- design/specs/facts/fact-ledger.sdp.md | 4 ++-- design/specs/obligations/do-nothing-check.sdp.md | 4 ++-- design/specs/obligations/obligation-module.sdp.md | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/design/README.md b/design/README.md index 7d4e81e..85bf2c3 100644 --- a/design/README.md +++ b/design/README.md @@ -274,7 +274,7 @@ Beyond the plan's list, three product decisions the packages surfaced: - The late-payment policy of `StartCheckout`: reallocate, ask a person, or refund as a separate operation ([`start-checkout-example`](specs/processes/start-checkout-example.sdp.md)). - Whether an old process run is kept, migrated or blocked on a given deploy ([`workflow-processes`](specs/processes/workflow-processes.sdp.md)). -Probes 1 to 5 ran on a native backend in slice S0 on 2026-10-01. Each result is recorded on its fact and on the Specs that rest on it, and [`probe-plan`](specs/facts/probe-plan.sdp.md) says what stays open: Probe 1 (the client guarantee holds across a restart, a closed client can lose a command, and the HTTP client does not retry, D6), Probe 2 (`ConvexError` data survives a nested mutation and a component boundary, F14, D7), Probe 3 (a component call costs about 2 ms more than a helper call on a local backend, with hosted cost and quota open, F4, D2, D8), Probe 4 (transaction limits are one budget across nested calls and components, F13, D10), Probe 5 (a parent query over a component query stays reactive and pinned pages stay contiguous, and the client hook loses rows after a capped split, F15, D8). Probe 6 ran on backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6 (the component drives parent-table batches and, through a parent mutation that returns the context's cursor, batches over a context's enumeration; a migration cannot walk another component's table; cancel, a thrown batch, a failed transaction and a restart resume from the saved cursor, F17, F1, D9). Probe 7 remains pending (what a restore leaves of scheduler, Workpool and Workflow state; the system table's states and retention, F12, F16, D13, D19). +Probes 1 to 5 ran on a native backend in slice S0 on 2026-10-01. Each result is recorded on its fact and on the Specs that rest on it, and [`probe-plan`](specs/facts/probe-plan.sdp.md) says what stays open: Probe 1 (the client guarantee holds across a restart, a closed client can lose a command, and the HTTP client does not retry, D6), Probe 2 (`ConvexError` data survives a nested mutation and a component boundary, F14, D7), Probe 3 (a component call costs about 2 ms more than a helper call on a local backend, with hosted cost and quota open, F4, D2, D8), Probe 4 (transaction limits are one budget across nested calls and components, F13, D10), Probe 5 (a parent query over a component query stays reactive and pinned pages stay contiguous, and the client hook loses rows after a capped split, F15, D8). Probe 6 ran on backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6 (the component drives parent-table batches and, through a parent mutation that returns the context's cursor, batches over a context's enumeration; a migration cannot walk another component's table; cancel, a thrown batch, a failed transaction and a restart resume from the saved cursor, F17, F1, D9). Probe 7 has local native observations for scheduler states, short retention, argument accounting, scan cost and both CLI replacement modes; hosted dashboard restore, seven-day expiry, Workpool and Workflow remain open (F12, F13, F16, D13, D19). | Tier | Fact | Subject | |---|---|---| diff --git a/design/specs/application/restore.sdp.md b/design/specs/application/restore.sdp.md index d69fd6a..21f2b8d 100644 --- a/design/specs/application/restore.sdp.md +++ b/design/specs/application/restore.sdp.md @@ -52,7 +52,7 @@ The parts of restore that concern obligations, external dispatch and provider ev ### Open questions -- [non-blocking] Probe 7 pending: what a restore leaves of component state decides whether the contexts' tables come back with the parent's; the drill on the Orders and Inventory application shows it for Layer 2 and Probe 7 shows it for Workpool and Workflow before Layer 3 (Probe 7, F12, D19) +- [non-blocking] Probe 7 observes local CLI replacement restoring the parent, Orders and Inventory documents with IDs and creation times; fresh replacement leaves scheduler tables empty and replacement in place keeps existing schedules; the complete Layer 2 restore drill, hosted dashboard restore, Workpool and Workflow remain open (Probe 7, F12, D19) - [non-blocking] Extension E-42: the doc says restore runs with matching code and configuration, that domain, journal and read-model invariants hold, and that restore is tested; it does not list the checks or the record, and it does not say what a restore does to a rebuild that was in flight at the backup. The option taken here: four checks, journal, domain, read model and receipt, each a bounded checkpointed batch, the stream check bounded through the `rebuild` query's `maxEvents` argument so a batch folds at most 500 events in all, each folded stream also reading its own stored baseline event, a fifth step that inventories the closed gate entries, the generations in `building`, `verifying` or `verified` and the `imports` rows in `running` that the snapshot restored, a `restoreRuns` row as the record, an accepted branch that resumes every stranded generation the operator did not abort, and the `MAINTENANCE_MODE` environment variable as the door (E-42, D19, F12) - [non-blocking] Extension E-42, the stream check's batch size: `limitCheckBatch` states the read of a batch that folds one stream, and each further stream the same batch folds reads its own stored baseline event of at most `budgetBytes + 4,096 + 61` bytes; the batch size that keeps a batch of several folded streams under the 16 MiB read ceiling is not pinned (E-42, F13) @@ -60,7 +60,7 @@ The parts of restore that concern obligations, external dispatch and provider ev - rule: Restore runs with the code and configuration the backup was taken under; the recovery plan names code, schemas, configuration, credentials, data and external outcome evidence (D19, Sc L2-8) - rule: Restore is tested, not only backup; the drill runs this workflow on a representative dataset in a disposable deployment before every release (D19, Sc L2-8) -- rule: A backup is a consistent snapshot of table data and excludes code, configuration, environment variables and pending scheduled functions; a restore wipes existing data and replaces it (F12, S8) +- rule: A backup excludes code, configuration, environment variables and scheduler rows; local CLI replacement replaces tables named in the archive, preserves destination environment variables, leaves a fresh scheduler empty and preserves an in-place destination scheduler; hosted dashboard restore is a separate unprobed path (F12, S8) - rule: Restore starts with dispatch off; in the transactional profile that is the writer door closed through the environment until the checks pass (D19, E-42) - rule: The checks never run a command, never repeat an external effect and never rewrite an event (Law 10, D5) - rule: Every check is a bounded, checkpointed batch below the transaction limits (D19, F13) @@ -72,7 +72,7 @@ The parts of restore that concern obligations, external dispatch and provider ev - The operator picks the backup and the recovery-plan entry recorded with it: commit, schema hash, configuration hash, credential locations and the location of external outcome evidence (D19, Sc L2-8) - The operator sets `MAINTENANCE_MODE` to `restore` on the target deployment, so the writer check refuses every write with `writePaused` from now on (E-42, D19) - The operator deploys the recorded commit and configuration and confirms that the deployed schema matches the backup's schema hash (D19, Sc L2-8) -- The operator restores the snapshot into the target from the dashboard or the CLI import; the restore is destructive and replaces the parent's table data and, as the drill must show, the components' (S8, F12, Probe 7) +- The operator restores the snapshot into the target with matching code; native CLI replacement preserves the observed parent, Orders and Inventory documents, including IDs and creation times, while keeping destination environment variables and any existing scheduler rows; the dashboard path must be proved separately (S8, F12, Probe 7) - The operator starts the run record with `startRestoreRun`, which stores the backup ID, the backup time, the commit, the hashes and the operator (E-42, D19) - The journal check runs in batches: for each stream of a `list` page taken with `includeDeleted`, the context's `rebuild` query, called with the events the batch may still fold as `maxEvents`, reports `version`, `savedVersion`, `fromBaseline` and `eventsApplied`, and the check asserts that `eventsApplied` equals `version` minus the baseline version and that `version` equals `savedVersion`, which shows the events since the latest baseline are contiguous and the stream row agrees; a `rebuild` answer refused with `tailTooLong` is recorded as a journal finding naming the stream and its tail length, never retried with a bigger batch, while one refused with `budgetExhausted` is not a finding but the point where this batch stops and the next one resumes; event identity is enforced by the append's indexed read and is not rescanned (E-42, E-24, E-25, D2, D5) - The domain check runs in the same batches from the same `rebuild` call: its `equal` flag shows that folding the events since the latest baseline through `evolve` gives the saved state at the stream version (E-42, D3, Law 3, Sc L0-2) diff --git a/design/specs/decisions/d13-deferred-work-is-an-obligation.sdp.md b/design/specs/decisions/d13-deferred-work-is-an-obligation.sdp.md index 72d5bbc..1bf8222 100644 --- a/design/specs/decisions/d13-deferred-work-is-an-obligation.sdp.md +++ b/design/specs/decisions/d13-deferred-work-is-an-obligation.sdp.md @@ -32,12 +32,12 @@ An obligation records one promise. It commits in the same transaction as the bus ### Open questions -- [non-blocking] Probe 7 pending, the do-nothing check at activation: a plain scheduled mutation plus a scan of `_scheduled_functions` for failed runs may cover a local reaction; the obligation table earns its cost through restore, since backups exclude scheduled functions, through retention past the system table's window, and through business visibility and operator exits; the probe confirms the system table's states and retention and what a restore leaves before Layer 3 is built (Probe 7, D13, F12, F16) +- [non-blocking] Probe 7 observes the local scheduled reaction, states, scan boundary and both CLI replacement modes; the owner must apply those measurements to the do-nothing check before activation; seven-day expiry, hosted dashboard restore, Workpool and Workflow remain unprobed (Probe 7, D13, F12, F16) ## Decision - context: The concern is deferred work: an effect after the transaction, a wait, background isolation or external I/O; Convex gives scheduling that commits with the mutation, scheduled mutations that run exactly once with internal retries, scheduled actions that run at most once, a system table of scheduled runs kept for 7 days, and backups that exclude pending scheduled functions (D13, F8, F9, F12, F16) -- alternative: Do nothing beyond Convex: a plain scheduled mutation plus a scan of `_scheduled_functions` for failed runs; it may cover a local reaction and is checked at activation by Probe 7; rejected for the durable profile because restore drops pending scheduled functions, the system table keeps results for 7 days only, and neither gives business visibility or operator exits (D13, F12, F16, Decision method rule 2) +- alternative: Do nothing beyond Convex: a plain scheduled mutation plus a scan of `_scheduled_functions` for failed runs; it may cover a local reaction and is checked at activation by Probe 7; rejected for the durable profile because fresh CLI replacement has no scheduled intent and in-place replacement can retain intent outside restored data, the system table keeps results for 7 days only, and neither gives business visibility or operator exits (D13, F12, F16, Decision method rule 2) - alternative: A copy of every event as an outbox that workers consume; rejected, because an obligation is not a copy of every event (D13) - alternative: An `onComplete` callback that decides whether the effect happened; rejected, because no callback decides and the obligation or an identified domain record is the evidence (D13, Law 7) - alternative: A specialized record per effect type; rejected as the default and kept only where a record carries distinct business evidence, such as payment attempts or approvals (D13) diff --git a/design/specs/decisions/d19-operations-travel-with-capability.sdp.md b/design/specs/decisions/d19-operations-travel-with-capability.sdp.md index 0b8bc02..a4646e8 100644 --- a/design/specs/decisions/d19-operations-travel-with-capability.sdp.md +++ b/design/specs/decisions/d19-operations-travel-with-capability.sdp.md @@ -33,7 +33,7 @@ Each installed capability carries its own operations: logging that never cancels ## Decision - context: The concern is operating what the layers create: logging, audit, restore, bounds, retention, diagnosis, release; Convex gives backups that exclude pending scheduled functions, per-transaction limits, a scheduled-functions table with 7-day retention, and dashboard logs that are observations, not authority (D19, F12, F13, F16) -- alternative: Do nothing beyond Convex: rely on the dashboard, its logs and its backups without design-level operations; rejected, because a restore drops pending scheduled functions, logs are observations and not authority, and a telemetry failure must never cancel a valid write (D19, F12, Decision method rule 2) +- alternative: Do nothing beyond Convex: rely on the dashboard, its logs and its backups without design-level operations; rejected, because fresh CLI replacement has no scheduled intent and in-place replacement can keep intent outside restored data, logs are observations and not authority, and a telemetry failure must never cancel a valid write (D19, F12, Decision method rule 2) - alternative: Telemetry that can veto a business write; rejected (D19, D18) - alternative: Reset-only upgrades; rejected for anything but disposable environments (D19) - alternative: Operations shipped with each capability as the rules below; this is the option chosen (D19) diff --git a/design/specs/facts/fact-ledger.sdp.md b/design/specs/facts/fact-ledger.sdp.md index f0492e9..d6fe7e5 100644 --- a/design/specs/facts/fact-ledger.sdp.md +++ b/design/specs/facts/fact-ledger.sdp.md @@ -39,8 +39,8 @@ Thirteen facts are documented and four are assumed as the doc states them. Four The four statuses map to the constraint target `evidence.status:documented`, `evidence.status:rechecked`, `evidence.status:probed` or `evidence.status:assumed`; `measurableBy` names the source and, for an assumed fact, the probe. -- factsDocumented: F1, F2, F7, F8, F9, F10, F11, F12, F13 (Fact ledger) -- factsRechecked: F3, F4, F5, F6 on 2026-09-29; F16 on 2026-09-30 against S6, changed from assumed (Fact ledger, F16) +- factsDocumented: F1, F2, F7, F8, F9, F10, F11, F12, F13; F12 retains this status for the wider backup and dashboard claim, with local CLI replacement observations recorded on the fact (Fact ledger) +- factsRechecked: F3, F4, F5, F6 on 2026-09-29; F16 on 2026-09-30 against S6, changed from assumed; native observations establish the five states and scan cost, while seven-day retention and expiry remain documented and not run (Fact ledger, F16) - factsAssumed: F17, with Probe 6; the doc also lists F14 and F15, which the corpus records as probed (Fact ledger, Probes) - factsProbed: F14 by Probe 2 and F15 by Probe 5, on 2026-10-01 on backend release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0; Probe 1 on F5, Probe 3 on F4 and Probe 4 on F13 ran the same day, and their results are recorded on those facts, whose status stays documented or rechecked (Fact ledger, Probes) - factsBeyondTheDoc: F18 by Probe 8, F19 by Probe 9 and Probe 11, and F20 by Probe 10 on native backend `precompiled-2026-09-28-5c7cb5b`, with `convex` 1.46.0 and `convex-helpers` 0.1.124; each states its package source and the limits of the local evidence (Fact ledger, F18, F19, F20) diff --git a/design/specs/obligations/do-nothing-check.sdp.md b/design/specs/obligations/do-nothing-check.sdp.md index e51783a..6058f91 100644 --- a/design/specs/obligations/do-nothing-check.sdp.md +++ b/design/specs/obligations/do-nothing-check.sdp.md @@ -30,12 +30,12 @@ Before the obligation module is built, the do-nothing option is checked against ### Open questions -- [non-blocking] Probe 7 pending: the probe must show, on a native backend, that `_scheduled_functions` exposes Pending, InProgress, Success, Failed and Canceled with the 7-day retention; what a backup and restore leave of pending scheduled functions, Workpool jobs and Workflow runs; whether a scan of the system table for one function's failed runs is bounded by the 32,000 documents scanned per transaction when many functions are outstanding; and whether the restored deployment can reconstruct a local reaction's intent from the system table alone (Probe 7, F12, F16, F13) +- [non-blocking] Probe 7 observes a plain scheduled local reaction and all five states on the pinned native backend; a failed-function scan counts zero documents and bytes but exhausts the 4096-read allowance at 4092 total rows; fresh CLI replacement leaves no scheduler intent, while replacement in place preserves later intent outside restored data; seven-day expiry, hosted dashboard restore, Workpool and Workflow remain open, and the owner rules on D13 from these observations (Probe 7, F12, F16, F13) ## Decision - context: The concern is a local reaction that must happen after a transaction; Convex gives a scheduled mutation that commits with the caller and runs exactly once, a system table that shows its state for 7 days, and backups that exclude pending scheduled functions (D13, F8, F9, F12, F16) -- alternative: Do nothing beyond Convex: schedule the reaction as a plain mutation from the command and scan `_scheduled_functions` for failed runs; checked first against Sc L3-1 to Sc L3-4 and Sc L3-7, Sc L3-8; it passes Sc L3-1 through exactly-once execution and fails Sc L3-7 because a restore leaves no pending scheduled function to rerun, fails Sc L3-8 because the system table forgets a run after 7 days, and fails Sc L3-4 because a failed run has no operator exit and no business meaning (D13, F9, F12, F16, Decision method rule 2) +- alternative: Do nothing beyond Convex: schedule the reaction as a plain mutation from the command and scan `_scheduled_functions` for failed runs; checked first against Sc L3-1 to Sc L3-4 and Sc L3-7, Sc L3-8; it passes Sc L3-1 through exactly-once execution and fails Sc L3-7 because fresh CLI replacement leaves no pending scheduled function to rerun and in-place replacement can retain intent outside restored data, fails Sc L3-8 because the system table forgets a run after 7 days, and fails Sc L3-4 because a failed run has no operator exit and no business meaning (D13, F9, F12, F16, Decision method rule 2) - alternative: A copy of every event consumed by workers, an outbox; rejected because an obligation is not a copy of every event (D13) - alternative: The obligation table with one lifecycle, a wrapper, a sweeper, retention and operator operations; this is the option chosen for the durable profile, subject to Probe 7 (D13) - decision: The obligation module is activated on its trigger only after Probe 7 has run, and the check is made once for the durable profile, not per reaction: the do-nothing option is rejected for the profile because it fails Sc L3-4, Sc L3-7 and Sc L3-8 as the first alternative records, so every local reaction of an installed durable profile is an obligation and none stays a plain scheduled mutation; the table earns its cost through restore, retention past the system table's window, business visibility and operator exits (D13, Law 8, Probe 7) diff --git a/design/specs/obligations/obligation-module.sdp.md b/design/specs/obligations/obligation-module.sdp.md index 6523eee..7d0378e 100644 --- a/design/specs/obligations/obligation-module.sdp.md +++ b/design/specs/obligations/obligation-module.sdp.md @@ -45,12 +45,12 @@ The module lives in the parent as a shared library and two parent tables, becaus - assumption: Scheduling from a mutation commits with it, so the first dispatch exists exactly when the obligation exists (F8) - assumption: A scheduled mutation runs exactly once with internal errors retried and developer errors ending it, so a failed wrapper leaves the obligation pending for the sweeper (F9) - assumption: A nested `ctx.runMutation` rolls back on throw and the wrapper can catch and continue, so the wrapper can record an attempt after the body failed (F3) -- assumption: Backups exclude pending scheduled functions, so a restored deployment has obligations but no dispatches (F12) +- assumption: Backups exclude scheduled functions; a CLI replacement into a fresh backend restores obligations without dispatches, while a replacement in place keeps every dispatch the destination had, including those created after the export (F12) - assumption: `_scheduled_functions` shows the states Pending, InProgress, Success, Failed and Canceled for 7 days after completion (F16) ### Open questions -- [non-blocking] Probe 7 pending: before Layer 3 is built the probe must confirm the states and retention of `_scheduled_functions` on a native backend and show what a restore leaves of scheduler, Workpool and Workflow state; a plain scheduled mutation plus a system-table scan may still cover local reactions, and `spec:obligations.do-nothing-check` names what the probe must show (Probe 7, D13, F12, F16) +- [non-blocking] Probe 7 observes local scheduler states, short retention, both CLI replacement modes, scheduler-ID validation and the failed-function scan boundary; a stored `v.id("_scheduled_functions")` can validate after fresh replacement while resolving to no row, so it is a typed reference and not proof of a live dispatch; seven-day expiry, hosted restore, Workpool and Workflow remain open, and the owner rules on the do-nothing check (Probe 7, D13, F12, F16) - [non-blocking] Extension E-50: the doc does not say where the module lives; this design places it in the parent as a library plus the `obligations` and `obligationRepairs` tables with a static handler registry keyed by handler key and version, because the wrapper runs parent-side bodies as nested mutations and the record reuses the parent's actor and scope types; the alternative, a Convex component driven by function handles like Workpool, is not pre-empted (D13, D11, E-50) - [non-blocking] OQ1 dependency: if the first experiment shows that a trivial context may be plain tables, a local reaction's body still runs as a nested mutation and nothing here changes; if component call cost breaks the read budgets, only the bodies change shape (OQ1, Probe 3) - [non-blocking] Decision method rule 4: the doc says Layers 0 to 2 get full detail, the first experiment runs, and Layer 3 is written from what it showed; this module and its children are written now, at the depth D13 to D15 give, with numeric defaults the doc does not give, the backoff and attempt budget of E-51, the sweeper's cadence and bounds of E-52, the fan-out bounds of E-54, the retention horizons of E-55, the lease and settle repeats of E-58 and the index set of E-57; every such number is a provisional default until the experiment's measurements and Probe 7 confirm or replace it, and the owner confirms that writing the structure now does not pre-empt the experiment (Decision method rule 4, D13, Probe 7, E-51, E-52, E-54, E-55, E-57, E-58) From 9774ec16e7e9f900a0ac9ffc0a334dd661b2e4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:57:24 +0200 Subject: [PATCH 044/155] design: the mutation timeout bounds a function's own computation, not its call tree Co-Authored-By: Claude Opus 5.5 --- design/specs/facts/f13-transactions-have-limits.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/facts/f13-transactions-have-limits.sdp.md b/design/specs/facts/f13-transactions-have-limits.sdp.md index 6f48711..b5d32b9 100644 --- a/design/specs/facts/f13-transactions-have-limits.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.sdp.md @@ -31,7 +31,7 @@ The documented numbers bound batches. Probe 4 observes transaction budgets; Prob - limitTransactionRead: 16 MiB read, 32,000 documents scanned and 4,096 index ranges read per transaction (F13, S9) - limitTransactionWrite: 16 MiB written and 16,000 documents written per transaction (F13, S9) - limitFunctionPayload: 16 MiB per function argument set and 16 MiB per return value; a Node action's arguments are capped at 5 MiB (F13, S9) -- limitQueryMutationTimeout: 1 second of execution per query or mutation (F13, S9) +- limitQueryMutationTimeout: 1 second of execution per query or mutation, a limit on the function's own computation and not on the elapsed time of its call tree, because nested calls draw on a separate system-operation budget (F13, S9, F19) - limitActionTimeout: 30 minutes in the Convex runtime and 10 minutes in the Node runtime (F13, S9) - limitScheduling: the limits page states 1000 scheduled functions per mutation, 4 MiB per scheduled argument set, 16 MiB summed arguments per mutation and 1000000 outstanding functions; the scheduling page states 8 MB total, both read 2026-10-02; the design keeps 4 MiB per call as its ceiling because this release names that limit as a future hard error (F13, S9, Probe 7) - limitSchema: 32 indexes per table, 16 fields per index, 10,000 tables per deployment (F13, S9) From fba155a849b488b3f430e8fbb83f9cf732caab01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:57:45 +0200 Subject: [PATCH 045/155] design: the ledger finding for schedules a restore in place keeps, and the in-place half on the import finding Co-Authored-By: Claude Opus 5.5 --- design/reviews/consensus-ledger.json | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index f4f3497..f3fc1fe 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -4690,7 +4690,7 @@ "issue": "`spec:application.restore`, near line 70, says a restore drops every pending scheduled function, and F12, near line 13, infers that no dispatches survive; on the pinned local backend a `convex import --replace` into the same backend kept every existing `_scheduled_functions` row, pending ones included, and into a fresh backend left the table empty, so the sentence holds for a fresh target and not for an import in place, and the dashboard restore the backup page describes was not run.", "fix": "State the two cases apart in the restore Spec and on F12, and name which one the restore procedure is.", "status": "open", - "note": "Found by the import scout of lane 1d on the native backend, local backend at the pinned release; it enters the ledger through lane 1c, the ledger's one writer in the round. Shown at the native backend tier by `tests/native/snapshot-replacement.test.ts` at commit d53c651 on unit/import-scout; the run and the commands are in report-d-import-scout.md.", + "note": "Found by the import scout of lane 1d on the native backend, local backend at the pinned release; it enters the ledger through lane 1c, the ledger's one writer in the round. Shown at the native backend tier by `tests/native/snapshot-replacement.test.ts` at commit d53c651 on unit/import-scout; the run and the commands are in report-d-import-scout.md. Probe 7 shows the in-place half on the native backend at the pinned release: the exported documents replace later changes while every scheduler row, including ones created after the export, is kept, and a kept pending mutation still runs against the restored data (`tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts`). The restore Spec's step and its backup rule now state both cases; its rule near line 70 still says a restore drops every pending scheduled function, and the rule that replaces it is the restore build's, in `probe-7-architecture-restore-kept-schedules-not-reconciled`.", "slice": "S5", "detector": "native run of the import scout" }, @@ -4727,6 +4727,23 @@ "note": "S3 measurement on the native backend, 2026-10-02. The main thread ruled no command-path check and sent it to the unit that builds the restore check and `rebuildStream`. The reproduction stays as a test that states what happens. The build must show: on the native backend or at convex-test, a stock item whose current state admin access set ahead of its events is reported by the restore's domain check and by `rebuildStream` as a state that differs from its fold, while a stream whose state is its fold reports none; and the present reproduction still states what the command path does.", "slice": "S5", "detector": "native backend" + }, + { + "round": 3, + "id": "probe-7-architecture-restore-kept-schedules-not-reconciled", + "lens": "architecture", + "severity": "major", + "files": [ + "specs/application/restore.sdp.md", + "specs/obligations/retention-and-restore.sdp.md", + "specs/obligations/obligation-module.sdp.md" + ], + "issue": "The restore and its schedule rebuild assume a restore leaves no scheduled function. On the native backend at the pinned release that holds only for a CLI replacement into a fresh backend, where every document returns with its id and creation time, every `_scheduled_functions` table is empty, and a stored scheduler id still validates while `system.get` returns null. In a replacement in place the exported documents replace later changes while every scheduler row, including ones created after the export, is kept, and a kept pending mutation still runs against the restored data. So `spec:application.restore` near line 70 (a restore drops every pending scheduled function, and the accepted branch resumes each inventoried generation), `spec:obligations.retention-and-restore` near line 45 (schedules are rebuilt from obligations) and `rebuildStep2` near line 68 (running obligations whose in-flight action was lost with the backup), and the rearming that `spec:obligations.obligation-module` relies on, can rearm work beside a kept schedule that still runs.", + "fix": "Decide in the restore's build which replacement the restore procedure uses, and for a replacement in place a rule that reconciles or fences the kept schedules before anything is rearmed: the probe proposed that the fifth step resumes each inventoried generation under a reconciled fence, that schedules are reconciled with restored obligations and obsolete ones cancelled or fenced before new attempt IDs are minted, and that `rebuildStep2` also covers an action still in progress on an in-place destination. None of these is applied; each is a rule, not an observation.", + "status": "open", + "note": "Probe 7 on the native backend, local backend at the pinned release, 2026-10-02 (`report-probe-7-bindings.md`, the proposed lines for `restore.sdp.md:70`, `retention-and-restore.sdp.md:45` and `:68`, and the reconciling clause of `obligation-module.sdp.md:48`). The main thread ruled that the rule-adding lines go to the unit that builds the restore. The build must show, on the native backend: after a replacement into a fresh backend, a generation in `building` and a running obligation are resumed or rearmed once with no kept schedule; after a replacement in place with a pending batch and a pending dispatch kept, including one scheduled after the export, nothing runs twice and nothing runs against a fence or attempt the restore superseded; and which replacement the procedure uses is stated and tested.", + "slice": "S5", + "detector": "native backend" } ], "sessions": [ From 0bd111ebda43071052deb150ea507b3bcf6efc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:57:57 +0200 Subject: [PATCH 046/155] Correct gate validator type assertions --- tests/pure/gate.test.ts | 8 ++++++-- tests/types/gate-audit.test-d.ts | 14 +++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/pure/gate.test.ts b/tests/pure/gate.test.ts index 516c4db..081fdd4 100644 --- a/tests/pure/gate.test.ts +++ b/tests/pure/gate.test.ts @@ -180,7 +180,11 @@ test("pure: registered gate entries export the exact argument and return validat isInternal: boolean; }; expect(exported.isInternal).toBe(true); - expect(JSON.parse(exported.exportArgs())).toEqual(args.json); - expect(JSON.parse(exported.exportReturns())).toEqual(returns.json); + expect(JSON.parse(exported.exportArgs())).toEqual( + (args as unknown as { json: unknown }).json, + ); + expect(JSON.parse(exported.exportReturns())).toEqual( + (returns as unknown as { json: unknown }).json, + ); } }); diff --git a/tests/types/gate-audit.test-d.ts b/tests/types/gate-audit.test-d.ts index b8bd1b6..0eb787d 100644 --- a/tests/types/gate-audit.test-d.ts +++ b/tests/types/gate-audit.test-d.ts @@ -4,7 +4,7 @@ import type { FunctionArgs, FunctionReturnType, FunctionReference, - PaginationOptions, + paginationOptsValidator, PaginationResult, SchemaDefinition, } from "convex/server"; @@ -111,16 +111,14 @@ test("compiled: the fixture exposes the pinned internal gate entries", () => { expectTypeOf< FunctionReturnType >().toEqualTypeOf(); - expectTypeOf>().toEqualTypeOf< - Record - >(); + expectTypeOf>().toBeNever(); expectTypeOf>().toEqualTypeOf<{ restore: boolean; closed: ClosedEntry[]; }>(); expectTypeOf>().toEqualTypeOf<{ scopeKey: string; - paginationOpts: PaginationOptions; + paginationOpts: Infer; }>(); expectTypeOf>().toEqualTypeOf< PaginationResult @@ -156,16 +154,14 @@ test("compiled: the example exposes the pinned internal gate entries", () => { expectTypeOf< FunctionReturnType >().toEqualTypeOf(); - expectTypeOf>().toEqualTypeOf< - Record - >(); + expectTypeOf>().toBeNever(); expectTypeOf>().toEqualTypeOf<{ restore: boolean; closed: ClosedEntry[]; }>(); expectTypeOf>().toEqualTypeOf<{ scopeKey: string; - paginationOpts: PaginationOptions; + paginationOpts: Infer; }>(); expectTypeOf>().toEqualTypeOf< PaginationResult From 1b92bd8eefdc0a5894d8df6c92b4066f610e66da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Fri, 2 Oct 2026 23:58:05 +0200 Subject: [PATCH 047/155] design: the scheduler and restore probe's result on the rows of the restore, the do-nothing check and the scheduler Co-Authored-By: Claude Opus 5.5 --- design/decisions/register.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/design/decisions/register.json b/design/decisions/register.json index f1f2bf2..4de1d90 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -3252,7 +3252,8 @@ "reason": "Delegated test: which mechanism the restore needs for the contexts' tables turns on a probed fact; owner test 1 would hold only if the probe showed a snapshot that leaves component tables behind, which would set F12 against the doc; the probe's own design is the convex advisor's.", "lean": { "do": "Keep the restore workflow's reading that the snapshot replaces the components' table data, shown by the drill before it is relied on.", - "settles": "Probe 7 on a native backend and the Layer 2 drill on the Orders and Inventory application, as [design/specs/application/restore.sdp.md:55] and [design/specs/application/restore.sdp.md:75] name them." + "settles": "Probe 7 on a native backend and the Layer 2 drill on the Orders and Inventory application, as [design/specs/application/restore.sdp.md:55] and [design/specs/application/restore.sdp.md:75] name them.", + "restsOn": "Probe 7 result, native backend tier on the pinned local backend: a CLI replacement into a fresh backend restores the parent, Orders and Inventory documents with their ids and creation times and leaves every `_scheduled_functions` table empty, and a stored scheduler id still validates while `system.get` returns null; a replacement in place replaces later changes with the exported documents and keeps every scheduler row, including ones created after the export, and a kept pending mutation still runs against the restored data. The full Layer 2 restore drill was not run." }, "checkedBy": null, "fork": null, @@ -4676,7 +4677,7 @@ "reason": "Owner test 1: if Probe 7 lets the do-nothing option stand for the durable profile, D13's chosen option changes, which edits the doc, and [design/specs/obligations/do-nothing-check.sdp.md:45] has the owner restate readiness after the probe.", "lean": { "do": "Keep the obligation module as the durable profile's chosen option, with Probe 7 confirming F12 and F16 rather than reopening D13.", - "restsOn": "[design/specs/obligations/do-nothing-check.sdp.md:38] fails the do-nothing option on Sc L3-4, Sc L3-7 and Sc L3-8 from [design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md:11], documented, and [design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md:11], rechecked; what the probe adds is what a restore leaves of the scheduler, Workpool and Workflow [design/specs/obligations/do-nothing-check.sdp.md:33]. **Judgment.** A system table that forgets a run after 7 days gives the person on call no exit and no record, whatever the probe shows about restore.", + "restsOn": "[design/specs/obligations/do-nothing-check.sdp.md:38] fails the do-nothing option on Sc L3-4, Sc L3-7 and Sc L3-8 from [design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md:11], documented, and [design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md:11], rechecked; what the probe adds is what a restore leaves of the scheduler, Workpool and Workflow [design/specs/obligations/do-nothing-check.sdp.md:33]. **Judgment.** A system table that forgets a run after 7 days gives the person on call no exit and no record, whatever the probe shows about restore. Probe 7 result, native backend tier on the pinned local backend: a CLI replacement into a fresh backend restores the parent, Orders and Inventory documents with their ids and creation times and leaves every `_scheduled_functions` table empty, and a stored scheduler id still validates while `system.get` returns null; a replacement in place replaces later changes with the exported documents and keeps every scheduler row, including ones created after the export, and a kept pending mutation still runs against the restored data. A plain scheduled mutation performs the local reaction and a scan for failed runs works until the read boundary observed at 4,092 total rows; seven-day expiry, hosted dashboard restore, Workpool and Workflow are not run.", "settles": "Probe 7 on a native backend, then the owner's word on D13." }, "checkedBy": null, @@ -4968,7 +4969,7 @@ "reason": "Rule 3: a fact's status and its probe, settled by a probe [design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md:21]; the ruling the probe serves, D13's do-nothing check, is the operator's by the register.", "lean": { "do": "Run Probe 7's confirmation half in S5, the states and the 7-day retention of _scheduled_functions on the native backend, and its restore half before L3; the ruling on D13's do-nothing check is the operator's.", - "restsOn": "F16 is rechecked: the states and the 7-day retention are documented on S6, read 2026-09-30 [design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md:13,28]; the probe confirms them and shows what a restore leaves of scheduler, Workpool and Workflow state [design/specs/facts/probe-plan.sdp.md:64,73]; the backup page says nothing about component data or the scheduler [design/specs/facts/probe-plan.sdp.md:23]. **Judgment.** The confirmation half is cheap and local; the restore half needs a backup and a restore, and what a local backend cannot answer stays open on the fact by E-16's rule [design/specs/facts/probe-plan.sdp.md:57].", + "restsOn": "F16 is rechecked: the states and the 7-day retention are documented on S6, read 2026-09-30 [design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md:13,28]; the probe confirms them and shows what a restore leaves of scheduler, Workpool and Workflow state [design/specs/facts/probe-plan.sdp.md:64,73]; the backup page says nothing about component data or the scheduler [design/specs/facts/probe-plan.sdp.md:23]. **Judgment.** The confirmation half is cheap and local; the restore half needs a backup and a restore, and what a local backend cannot answer stays open on the fact by E-16's rule [design/specs/facts/probe-plan.sdp.md:57]. Probe 7 result, native backend tier on the pinned local backend: a CLI replacement into a fresh backend restores the parent, Orders and Inventory documents with their ids and creation times and leaves every `_scheduled_functions` table empty, and a stored scheduler id still validates while `system.get` returns null; a replacement in place replaces later changes with the exported documents and keeps every scheduler row, including ones created after the export, and a kept pending mutation still runs against the restored data. All five scheduler states are readable over a short interval, and the scheduled arguments of one mutation are refused above 16,777,216 bytes of Convex value bytes; seven-day expiry, hosted dashboard restore, Workpool and Workflow are not run.", "against": "A probe of documented behavior restates a page; the do-nothing option trusts S6 and spends the probe on the restore half only, the half D13 and D19 turn on.", "settles": "Probe 7 on the native backend at the pinned release, in S5 for the scheduler half and before L3 for the restore half. It should also read whether a v.id of _scheduled_functions validates after a restore, a lead, a pointer [design/STATE.md:293], and settle the two pages that disagree on a scheduled argument's size, 8 MB against 16 MiB, a pointer [design/STATE.md:325] against F13's 4 MiB per scheduled argument set [design/specs/facts/f13-transactions-have-limits.sdp.md:36].", "changes": "spec:facts.f16-scheduled-functions-table-shows-failed-runs, spec:facts.probe-plan, spec:obligations.obligation-module; S5 and L3. The alternative, no probe, edits the probe plan alone.", From 4ddbe2ed3220690bf00ec3766c17152af87d53eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:00:04 +0200 Subject: [PATCH 048/155] test: the race with a close reads each request's last execution from the log A writer that read the gate open and lost to the close re-runs, and the backend's log records both executions under one request ID. Co-Authored-By: Claude Opus 5.5 --- tests/native/gate.test.ts | 51 +++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/tests/native/gate.test.ts b/tests/native/gate.test.ts index a97c687..9eee011 100644 --- a/tests/native/gate.test.ts +++ b/tests/native/gate.test.ts @@ -251,6 +251,7 @@ test("native: two independent writers race a close and no successful command com const identifier = getFunctionName(api.depotCommands.createDocument); const closeIdentifier = getFunctionName(internal.gate.closeGate); let overlapping = 0; + let reruns = 0; for (let trial = 0; trial < 8; trial++) { const mark = await backend.admin.logMark(); const settled = await Promise.allSettled([ @@ -271,29 +272,42 @@ test("native: two independent writers race a close and no successful command com else await expectPaused(Promise.reject(result.reason), "tenant:t", "repair"); } - const records = await backend.admin.completionsSince( - mark, - (rows) => - rows.filter( - (row) => - row.componentPath === null && - (row.identifier === identifier || - row.identifier === closeIdentifier), - ).length >= 3, - ); + // Wait for the close, both requests, and as many refused executions as the clients saw refused. + const refused = settled + .slice(0, 2) + .filter((result) => result.status === "rejected").length; + const records = await backend.admin.completionsSince(mark, (rows) => { + const top = rows.filter((row) => row.componentPath === null); + const commandRows = top.filter((row) => row.identifier === identifier); + return ( + top.some((row) => row.identifier === closeIdentifier) && + new Set(commandRows.map((row) => row.requestId)).size >= 2 && + commandRows.filter((row) => row.error !== null).length >= refused + ); + }); const closeRecord = records.find( (row) => row.identifier === closeIdentifier && row.componentPath === null, ); expect(closeRecord).toBeDefined(); - const commands = records.filter( + // A writer that read the gate open and lost the race to the close re-runs (WP occSerialization), + // and the log shows each execution under the request's ID: the last one is the command's outcome. + const executions = records.filter( (row) => row.identifier === identifier && row.componentPath === null, ); + const lastByRequest = new Map(); + for (const row of executions) { + const last = lastByRequest.get(row.requestId); + if (last === undefined || row.timestamp > last.timestamp) + lastByRequest.set(row.requestId, row); + } + reruns += executions.length - lastByRequest.size; + const commands = [...lastByRequest.values()]; expect(commands).toHaveLength(2); - for (const command of commands) { + for (const command of commands) if (command.error === null) expect(command.timestamp).toBeLessThanOrEqual(closeRecord!.timestamp); - if (overlap(command, closeRecord!)) overlapping++; - } + for (const execution of executions) + if (overlap(execution, closeRecord!)) overlapping++; const cut = await stored(backend); await expectPaused( one.mutation( @@ -310,8 +324,13 @@ test("native: two independent writers race a close and no successful command com "commands overlapping the gate close by completion interval", overlapping, ); - // Without an overlapping invocation this run cannot witness the concurrency obligation. - expect(overlapping).toBeGreaterThan(0); + measure( + "command executions re-run after losing the race to the close", + reruns, + ); + // Without an execution overlapping the close, or one that lost the race to it and re-ran, this run + // cannot witness the concurrency obligation. + expect(overlapping + reruns).toBeGreaterThan(0); }); function overlap(a: CompletionRecord, b: CompletionRecord) { return ( From fcfd27b4c8f5da7fd8c48365de06354112c68fcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:00:37 +0200 Subject: [PATCH 049/155] design: the bound on an order line's stock item ID, its two examples, and README counts and lines that say what the tree holds PlaceOrder's declaration bounds each order line's stock item ID at 64 bytes of UTF-8, refused invalidInput at step 1, so the maximum order's OrderPlaced (11,725 bytes by getConvexSize) fits the 16,384-byte payload bound. Two examples under the first experiment: 100 lines at the bound applied, one byte past it refused. The stock contention term says what the contention run sends and records. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 23 +++++++++---- design/specs/application.pack.sdp.md | 2 ++ ...riment.order-at-stock-item-id-bound.sdp.md | 33 +++++++++++++++++++ .../specs/application/first-experiment.sdp.md | 8 ++++- ...nt.stock-item-id-past-bound-refused.sdp.md | 33 +++++++++++++++++++ .../orders-inventory-example.sdp.md | 9 ++--- 6 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 design/specs/application/first-experiment.order-at-stock-item-id-bound.sdp.md create mode 100644 design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md diff --git a/design/README.md b/design/README.md index 5e5717e..ec7b122 100644 --- a/design/README.md +++ b/design/README.md @@ -2,7 +2,7 @@ ## What this is -The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 223 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. +The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 225 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. The thesis, in the doc's words. Ownership follows bounded contexts. Atomicity follows the business operation. Asynchrony follows a concrete need to defer work. The default operation is one Convex mutation that authorizes a command, makes the domain decision, saves current state with its events, updates essential read models and records the outcome. A second transaction exists only to wait, to spread load, or to reach an external system. The design succeeds when adding domain sophistication does not add infrastructure. @@ -45,11 +45,11 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -223 specs · 5 packs · 121 anchors → 349 nodes · 1186 edges (0 errors, 0 warnings) +225 specs · 5 packs · 121 anchors → 351 nodes · 1192 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` -Any warning and any error is a defect. Every example declares `verifies` toward its parent for fidelity to the acceptance contract, and no test anchor binds it yet. The Protocol reads an unbound example below `ready` as data and does not warn; recipe 10 lists each one as a declared verifier that is not enabled. `npx sdp validate --watch` is the authoring loop. +Any warning and any error is a defect. Every example declares `verifies` toward its parent for fidelity to the acceptance contract. An example that a test's `specTest` anchor binds gives its parent a verifier; an example that no anchor binds is data below `ready`, and the Protocol does not warn; recipe 10 lists each such example as a declared verifier that is not enabled. `npx sdp validate --watch` is the authoring loop. ## The component map @@ -85,7 +85,7 @@ Detail follows the build (decision method, rule 4). Full means interfaces, TypeS | all | `platform`, `laws`, `decisions`, `facts` | [`foundation`](specs/foundation.pack.sdp.md) | 93 | | 0 | `kernel` | [`kernel-and-context`](specs/kernel-and-context.pack.sdp.md) | 7 | | 1 | `context`, `command` | [`kernel-and-context`](specs/kernel-and-context.pack.sdp.md) (19 with the kernel), [`command-pipeline`](specs/command-pipeline.pack.sdp.md) | 12 + 24 | -| 2 | `application`, `operations`, `constraints` | [`application`](specs/application.pack.sdp.md) | 44 | +| 2 | `application`, `operations`, `constraints` | [`application`](specs/application.pack.sdp.md) | 46 | | 3 | `obligations`, `effects` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 22 (of the pack's 43) | | 4 | `processes` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 6 | | 5 | `agents` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 7 | @@ -224,7 +224,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed, with `AuditRecordInput` and the source of each field in the command; the `operatorAudit` table for the gate changes an operator entry makes; Convex's own audit logging recorded as the do-nothing option; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | | E-44 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | `BusinessFailurePolicy` (`carry` or `reject`) per context call, applied in the executor body through the pinned helper `carryOrReject` | | E-45 | [`application.read-models`](specs/application/read-models.sdp.md) | At most 8 component reads per parent query and 4 read-model rows written per command, counted as the rows step 9 inserts, replaces or deletes and checked after each `streams` entry is applied, above which step 9 throws a plain error | -| E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a maximum of 100 lines, the bound `PlaceOrder`'s declaration carries; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` and `CancelOrder` call Orders first; the stock item keeps totals only and `CancelOrder` releases the cancelled order's own lines; `orderNotFound`, `orderAlreadyCancelled` and `insufficientAllocation` as `CancelOrder`'s rejections; the order allocation history view that Sc L2-6 builds | +| E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a maximum of 100 lines and a stock item ID of at most 64 bytes on each line, the bounds `PlaceOrder`'s declaration carries; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` and `CancelOrder` call Orders first; the stock item keeps totals only and `CancelOrder` releases the cancelled order's own lines; `orderNotFound`, `orderAlreadyCancelled` and `insufficientAllocation` as `CancelOrder`'s rejections; the order allocation history view that Sc L2-6 builds | | E-47 | [`application.first-experiment`](specs/application/first-experiment.sdp.md) | The `Measurement` record, one per measured run, and how each count is captured: commits from the backend's function log, documents and bytes from the `usageStats` of the command's top-level completion record, the calls per context from a pure test of the use case's executor because the log holds no record of a component call inside a mutation, and a count the backend does not expose recorded as a gap; the documents one `PlaceOrder` reads, expected near N + 6 and counted 3N + 5 on the pinned backend, left for the measurement run to explain | | E-48 | [`command.actor-and-scope`](specs/command/actor-and-scope.sdp.md) | The operator entry: an internal function that only admin access reaches, a required `operator` string of 1 to 512 bytes checked by `assertOperator`, no grant read, and the stated operator recorded on the generation row, the gate's entry and audit record, or the restore's run record; a stated operator proves no identity | | E-49 | [`command.actor-and-scope`](specs/command/actor-and-scope.sdp.md) | The tenant list: a `tenants` table written by `insertGrant` with a tenant's first grant and never deleted, read one tenant at a time through `nextTenant` in tenant-ID order by every pass over the whole deployment | @@ -338,7 +338,9 @@ The Layer 2 design added two more for the same reason, and nine examples that ve | none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-operator-cancel` | Probe 6: an operator cancels a committed successor | | none | `spec:facts.f17-migrations-fits-generation-backfill.probe-6-transaction-failure` | Probe 6: a transaction failure preserves the saved cursor | -Package A therefore has 93 Specs, Package C has 17 examples, Package D has 26, and the corpus has 60 examples that bind a doc row. +The measurement of the first experiment added `spec:application.first-experiment.native-contention`, a second example of Sc L2-9 for its concurrency half, as the scenario table shows. + +Package A therefore has 93 Specs, Package C has 17 examples, Package D has 20, and the corpus has 60 examples that bind a doc row. The example domain's second lifecycle command added six examples that verify no doc row, each a child of `spec:application.orders-inventory-example` under E-46: @@ -351,7 +353,14 @@ The example domain's second lifecycle command added six examples that verify no | none | `spec:application.orders-inventory-example.cancel-second-context-rejects` | verifies that a release Inventory refuses rolls back the cancel Orders recorded | | none | `spec:application.orders-inventory-example.cancel-without-grant-refused` | verifies that `CancelOrder` requires its own permission | -Package D therefore has 26 examples. +The bound on an order line's stock item ID added two examples that verify no doc row, each a child of `spec:application.first-experiment` under E-46: + +| Sc | Added example | Reason | +|---|---|---| +| none | `spec:application.first-experiment.order-at-stock-item-id-bound` | verifies that the maximum order with every stock item ID at its bound is applied and its `OrderPlaced` fits the payload bound | +| none | `spec:application.first-experiment.stock-item-id-past-bound-refused` | verifies that a stock item ID one byte past its bound is refused `invalidInput` before any read | + +Package D therefore has 28 examples. ## Integration notes diff --git a/design/specs/application.pack.sdp.md b/design/specs/application.pack.sdp.md index da49c73..884ec62 100644 --- a/design/specs/application.pack.sdp.md +++ b/design/specs/application.pack.sdp.md @@ -25,6 +25,8 @@ specs: - spec:application.first-experiment.order-of-1-line - spec:application.first-experiment.order-of-10-lines - spec:application.first-experiment.order-of-max-lines + - spec:application.first-experiment.order-at-stock-item-id-bound + - spec:application.first-experiment.stock-item-id-past-bound-refused - spec:application.first-experiment.native-acceptance-production-configuration - spec:application.first-experiment.native-contention - spec:application.orders-inventory-example diff --git a/design/specs/application/first-experiment.order-at-stock-item-id-bound.sdp.md b/design/specs/application/first-experiment.order-at-stock-item-id-bound.sdp.md new file mode 100644 index 0000000..4a9ef52 --- /dev/null +++ b/design/specs/application/first-experiment.order-at-stock-item-id-bound.sdp.md @@ -0,0 +1,33 @@ +--- +id: spec:application.first-experiment.order-at-stock-item-id-bound +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.first-experiment + verifies: spec:application.first-experiment +--- +# An order of the maximum number of lines at the stock item ID bound + +E-46 · native tier · production composition · no acceptance row; the example verifies that the maximum order fits the event payload bound when every stock item ID is at the bound `PlaceOrder`'s declaration carries. + +## Intent + +- outcome: An order of 100 lines whose every stock item ID is 64 bytes of UTF-8 is applied, and the `OrderPlaced` it records measures 11,725 bytes, under the 16,384-byte payload bound. (E-46, E-12, OQ3) + +```gwt +Given the production composition on a native backend +And an order of {size: "the maximum"} with stock contention {contention: "absent"} +And the order's stock item IDs are {idBytes: 64} bytes of UTF-8, the last line's {lastIdBytes: 64} +And the backend runs with {configuration: "production configuration"} +When {run: "the PlaceOrder use case"} runs +Then the caller receives {answer: "the result"} +And the command read {readDocuments: 305} documents and wrote {writtenDocuments: 204} +And the order's OrderPlaced payload measures {payloadBytes: 11725} bytes +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- After grants and the order summary's first rebuild through to its switch as setup, the test receives stock for the order's 100 stock items through `ReceiveStock`, each ID distinct and exactly 64 bytes of UTF-8, and sends one `PlaceOrder` with one line on each from an ordinary client under a request key. +- It asserts the applied response, reads the documents read and written from the command's own top-level completion record's `usageStats`, and measures with `getConvexSize` the payload of the one `OrderPlaced` event the Orders context's journal holds for the order, read through admin access. diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index b44d365..ad88fa8 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -37,7 +37,7 @@ relations: --- # The first experiment -Layer 2 · Detail: full · Traces: First experiment, Acceptance scenarios, Law 1, D3, D8, D9, D10, D11, F4, F13, Probe 3, OQ1, OQ3, OQ4, Sc L2-3, Sc L2-9, E-8, E-13, E-15, E-17, E-37, E-46, E-47. +Layer 2 · Detail: full · Traces: First experiment, Acceptance scenarios, Law 1, D3, D8, D9, D10, D11, F4, F13, Probe 3, OQ1, OQ3, OQ4, Sc L2-3, Sc L2-9, E-8, E-12, E-13, E-15, E-17, E-37, E-46, E-47. The first experiment builds Layers 0 to 2 in a small, clean application, Orders and Inventory, and measures what the design costs. It passes when every Layer 0, 1 and 2 scenario passes on a native backend. It measures orders of 1 line, 10 lines and the maximum of 100 lines, without and with stock contention, and counts top-level commits, function and component calls, documents read and written, and rows left behind, with healthy-path and retry costs reported separately. Semantics come first and speed second; the latency and throughput targets are set before the benchmark, for the pinned local backend, and no target changes after the run. The results feed the open question on component cost and the Layer 3 decisions. @@ -115,6 +115,7 @@ The experiment is an application, a test suite and a report. The application is - datasetSizes: 1 line, 10 lines and 100 lines, the maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) - contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with stock for one, so that one is applied and the rest are rejected for short stock or failed by the engine after its bounded retries, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry; the shared stock item is the first line of every order and each other line is on a stock item of that command alone, received once with quantity two, and no caller retries during the cell, while the caller may repeat a command the engine failed under its same request key (First experiment, Sc L1-11, F1, E-5, E-47) - limitOrderLines: 100 lines per `PlaceOrder`, the largest order the example supports and the bound its declaration carries, chosen under the 1 second timeout, the F13 ceilings and the adapter's stream and byte bounds; it is the example's promise and no platform limit, and the experiment reports documents read and written per line (OQ3, F13) +- limitStockItemIdBytes: [extension] 64 bytes of UTF-8 for the stock item ID of each order line, the bound `PlaceOrder`'s declaration carries beside its 100 lines and checks in its refinement, so a longer one is refused `invalidInput` at step 1 of the command pipeline, before any read and before either context is called, as the stock item ID bound of `spec:application.orders-inventory-example` states; it is chosen so that the maximum order's `OrderPlaced` fits the 16,384-byte payload bound of E-12: `getConvexSize` measures the payload at 25 + 100 × (53 + 64) = 11,725 bytes at the bound, where 128 bytes would make 18,125 and the 256 bytes of a stream ID 30,925 (E-46, E-12, OQ3, F13) - limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate read, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer (E-47, F13) - observedMeasurementPins: [extension] the measurement records name the native backend tier, the local backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124, `convex-test` 0.0.60, Node v24.19.0 and Linux 7.2.7-200.fc44.x86_64 on x64; these observations concern that local environment only (E-47, E-15, Sc L2-9) @@ -136,6 +137,7 @@ The experiment is an application, a test suite and a report. The application is ```gwt-vocabulary Given the production composition on a native backend And an order of {size:"1 line"|"10 lines"|"the maximum"} with stock contention {contention:"absent"|"present"} +And the order's stock item IDs are {idBytes:number} bytes of UTF-8, the last line's {lastIdBytes:number} And the contention matrix has sizes {sizes:string} and callers {callers:string} And the backend runs with {configuration:"production configuration"|"test configuration"} When {run:"the PlaceOrder use case"|"the end-to-end path"} runs @@ -145,6 +147,10 @@ And the use case makes {callsPerContext:number} call per context And the budgets {budgets:"hold"|"are exceeded"} And the contention matrix records one applied outcome per cell and {smallCellEngineFailures:number} engine failures at two and eight callers And authority, schemas and code path {parity:"match a release"|"differ from a release"} +And the caller receives {answer:"the result"|"the rejection invalidInput"} +And the rejection names line {line:number} with a stock item ID of {length:number} bytes against a bound of {limit:number} +And the command read {readDocuments:number} documents and wrote {writtenDocuments:number} +And the order's OrderPlaced payload measures {payloadBytes:number} bytes ``` ## Verification — reviewed diff --git a/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md b/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md new file mode 100644 index 0000000..f3d8ca5 --- /dev/null +++ b/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md @@ -0,0 +1,33 @@ +--- +id: spec:application.first-experiment.stock-item-id-past-bound-refused +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.first-experiment + verifies: spec:application.first-experiment +--- +# A stock item ID one byte past its bound is refused + +E-46 · native tier · production composition · no acceptance row; the example verifies that `PlaceOrder` refuses a stock item ID longer than its bound at step 1, before any read and before either context is called. + +## Intent + +- outcome: An order of 100 lines whose last stock item ID is 65 bytes of UTF-8 is refused `invalidInput` naming that line, having read and written no document, and nothing is stored. (E-46, E-12, D4) + +```gwt +Given the production composition on a native backend +And an order of {size: "the maximum"} with stock contention {contention: "absent"} +And the order's stock item IDs are {idBytes: 64} bytes of UTF-8, the last line's {lastIdBytes: 65} +And the backend runs with {configuration: "production configuration"} +When {run: "the PlaceOrder use case"} runs +Then the caller receives {answer: "the rejection invalidInput"} +And the rejection names line {line: 99} with a stock item ID of {length: 65} bytes against a bound of {limit: 64} +And the command read {readDocuments: 0} documents and wrote {writtenDocuments: 0} +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- After grants and the order summary's first rebuild through to its switch as setup, the test receives stock for all 100 stock items of the order through `ReceiveStock`, so the refusal is not short stock, and sends one `PlaceOrder` with one line on each from an ordinary client under a request key. +- It asserts the `ConvexError` data exactly, `{ kind: "rejection", code: "invalidInput", commandType: "PlaceOrder", message: "Line 99 needs a stock item ID of at most 64 bytes of UTF-8, not 65", details: { line: 99, length: 65, limit: 64 } }`, reads zero documents read and written from the command's own top-level completion record's `usageStats`, which places the refusal before the grant read and before either context's sub-transaction, and asserts that the receipts, both contexts' stream rows and events and the order summaries are unchanged. diff --git a/design/specs/application/orders-inventory-example.sdp.md b/design/specs/application/orders-inventory-example.sdp.md index f6262e3..cabbb6b 100644 --- a/design/specs/application/orders-inventory-example.sdp.md +++ b/design/specs/application/orders-inventory-example.sdp.md @@ -17,7 +17,7 @@ relations: --- # Orders and Inventory, the example domain -Story · Detail: full · Traces: First experiment, Law 1, Law 9, D3, D4, D6, D7, D9, D10, D16, Sc L1-4, Sc L1-11, Sc L2-3, Sc L2-6, E-2, E-9, E-38, E-39, E-46. +Story · Detail: full · Traces: First experiment, Law 1, Law 9, D3, D4, D6, D7, D9, D10, D16, Sc L1-4, Sc L1-11, Sc L2-3, Sc L2-6, E-2, E-9, E-12, E-38, E-39, E-46. The first experiment's domain has two bounded contexts, Orders and Inventory, one composed use case that makes them meet, one more lifecycle command, one command that creates stock, and one essential summary. The terms below are the ones the experiment's code, tests and measurements use. Where the doc names a thing, its word is kept; the second lifecycle command, the command that creates stock, the summary and the rejection reasons are the design's choices under E-46. `StartCheckout` from D16 is not part of the experiment; it is the Layer 4 contrast to atomic `PlaceOrder`. @@ -30,7 +30,7 @@ The second lifecycle command is `CancelOrder`. The stock item keeps two totals a ### Open questions -- [non-blocking] Extension E-46: the second lifecycle command, the essential summary, the command that creates stock, the unit price an order line carries, the rejection reasons, the stream budgets of the two stream types, the order of the two context calls and the history-dependent view Sc L2-6 builds are choices the doc leaves open; two of them are decided, that the stock item keeps totals only and `release` takes its quantities from the cancelled order's own lines, and that the order allocation history view stays in the experiment; the owner may substitute the others, and the experiment's terms move with them (E-46, E-2, E-9, First experiment) +- [non-blocking] Extension E-46: the second lifecycle command, the essential summary, the command that creates stock, the unit price an order line carries, the bound on an order line's stock item ID, the rejection reasons, the stream budgets of the two stream types, the order of the two context calls and the history-dependent view Sc L2-6 builds are choices the doc leaves open; two of them are decided, that the stock item keeps totals only and `release` takes its quantities from the cancelled order's own lines, and that the order allocation history view stays in the experiment; the owner may substitute the others, and the experiment's terms move with them (E-46, E-2, E-9, First experiment) ## Model @@ -48,7 +48,7 @@ The second lifecycle command is `CancelOrder`. The stock item keeps two totals a - **place operation** — The Orders context's list-shaped operation `place({ orders: [{ orderId, lines }] })`, planning one order stream per order at expected version 0, so a second place of the same order ID is answered `entityExists` before its decider runs (D10, E-46, Sc L1-4). - **cancel operation** — The Orders context's list-shaped operation `cancel({ orders: [{ orderId }] })`, planning one `cancel` stream command per order with no expected version, so the order's own state decides it, and declaring `maxStreams` 32 as the place operation does; it returns `{ orders: [{ orderId, lines }] }`, the lines of each cancelled order taken from its DTO (D10, E-46). - **receive operation** — The Inventory context's list-shaped operation `receive({ items })`, planning one stock item stream per distinct stock item, which creates the stream of a stock item that has none and adds the quantity to the quantity on hand; it is the only way a stock item comes to exist (Law 1, D10, E-46). -- **PlaceOrder** — The composed use case that records an order in Orders and allocates every line in Inventory in one mutation, with one receipt and one outcome, calling Orders first because it creates the order, so a UI double submit is answered `entityExists` before Inventory decides, and Inventory second; it rejects when stock is short, and an order with no line is refused `invalidInput` before either context is called (D10, D4, E-39, Sc L1-4). +- **PlaceOrder** — The composed use case that records an order in Orders and allocates every line in Inventory in one mutation, with one receipt and one outcome, calling Orders first because it creates the order, so a UI double submit is answered `entityExists` before Inventory decides, and Inventory second; it rejects when stock is short, and an order with no line, or with a line whose stock item ID is longer than the stock item ID bound, is refused `invalidInput` at step 1 of the command pipeline, before any read and before either context is called (D10, D4, E-39, E-46, Sc L1-4). - **ReceiveStock** — The command that receives stock, one call to the Inventory context's receive operation with one receipt and one outcome; every stock item of the experiment, a test's and a measurement's included, is created through it (Law 1, D10, E-46). - **CancelOrder** — The second lifecycle command: the composed use case `cancelOrder`, input `{ orderId: string }`, that calls the Orders context's cancel operation once with the one order and then the Inventory context's release operation once with the order ID and the stock item and quantity of each line Orders returned for that order, in one mutation with one receipt and one outcome, and answers `{ orderId: string; released: { stockItemId: string; quantity: number }[] }`, the release operation's lines; Orders comes first because the order's state refuses a cancel before Inventory decides, and the quantities are read from what Orders returned in this mutation, never from a read model (D10, D4, Law 9, E-39, E-46). - **CancelOrder declaration** — `name: "CancelOrder"`, `contractVersion: 1`, `permission: { permission: "orders.cancel" }`, `writes` the Orders `order` and the Inventory `stockItem`, `readModels` the order summary from the Orders `order`, `rejections: ["orderNotFound", "orderAlreadyCancelled", "insufficientAllocation"]` and no `bounds`, because the input holds no list, the streams the command writes are its one order and the stock items of that order's lines, at most the 100 `PlaceOrder` admitted, and the order ID is bounded at 256 bytes as every stream ID is (D6, D10, E-38, E-46). @@ -64,11 +64,12 @@ The second lifecycle command is `CancelOrder`. The stock item keeps two totals a - **release above allocation rejection** — The rejection `insufficientAllocation` that the stock item decider returns for a release larger than the stock item's quantity allocated, and the Inventory context's adapter throws, with the message "Cannot release N when M are allocated" and the details `{ requested, allocated }`; a cancel of a placed order does not ask for one, because `PlaceOrder` allocated every line in the mutation that recorded the order and one order is cancelled once, and if the two contexts ever disagree `CancelOrder` lists the code and nothing commits (D4, D7, E-46). - **invalid quantity rejection** — The rejection `invalidQuantity`, which a decider returns and its context's adapter throws. The order decider returns it for a place whose line has a quantity that is not a safe whole number of at least 1 or a unit price that is not a safe whole number of at least 0, with the message "Line I needs a quantity of at least 1 and a unit price of at least 0, each a whole number" and the details `{ line, stockItemId, quantity, unitPrice }`, and for lines whose total passes the largest safe whole number, with the details `{ total }`; the stock item decider returns it for a receive, allocate or release of a quantity that is not a safe whole number of at least 1, with the message "A quantity must be a positive whole number, not N" and the details `{ quantity }` (D4, D7, E-46). - **stock limit rejection** — The rejection `stockLimitExceeded` that the stock item decider returns, and the Inventory context's adapter throws, for a receive that would take the quantity on hand past the largest safe whole number, with the message "Cannot receive N on top of the M on hand" and the details `{ quantity, onHand }`; `ReceiveStock` lists the code beside `invalidQuantity`, and nothing commits (D4, D7, E-46). -- **stock contention** — Two `PlaceOrder` commands for the same stock item sent concurrently; on the native backend both are applied when the stock covers both, and one is applied and one is rejected `insufficientStock` when it covers only one, and the experiment measures the retry cost of the loser (Sc L1-11, First experiment). +- **stock contention** — N `PlaceOrder` commands for the same stock item sent concurrently, as the first experiment's contention run sends them at N of 2, 8 and 32 with stock for one; on the native backend one is applied and every other is rejected `insufficientStock` or, past the engine's bounded retries, failed as a technical failure the caller may repeat, and the experiment records the retry cost of the losers apart from the healthy path (Sc L1-11, First experiment). - **order summary** — The essential summary: a per-entity read model of the parent with the order's status, line count, total and the time it was placed, `{ orderId, status: "placed" | "cancelled", lineCount, total, placedAt }`, written from the order's DTO by every command that writes the order, so `PlaceOrder` inserts the row with `placed` and `CancelOrder` replaces it with `cancelled`, and listed by status (E-46, D8). - **order allocation history** — A history-dependent cross-stream read model of the parent keyed by order ID, folded as a `HistoryProjection` from `OrderPlaced` in Orders and `StockAllocated` and `AllocationReleased` across the stock item streams the order touched, holding per line when it was allocated and released; it is bound once per source stream type, rebuilds only under the write pause, and exists in the experiment for Sc L2-6, not as an essential read (E-9, E-46, D9, Sc L2-6). - **journal inspection** — The context queries that return a stream's events and its current version for a reviewer, used by the experiment and the restore checks (First experiment, D2). - **maximum order size** — The largest number of lines `PlaceOrder` accepts, 100, the bound its declaration carries: the example's promise and no platform limit, chosen by the command's author under the adapter's stream and byte ceilings (OQ3, E-46). +- **stock item ID bound** — The longest stock item ID an order line of `PlaceOrder` may carry, 64 bytes of UTF-8, the bound `PlaceOrder`'s declaration carries beside its 100 lines and checks in its refinement: an order whose line has a longer stock item ID is refused `invalidInput` at step 1 of the command pipeline, before any read and before either context is called, with the message "Line I needs a stock item ID of at most 64 bytes of UTF-8, not N" and the details `{ line, length, limit }` of the first such line, `line` counting from 0; the bound exists because `OrderPlaced` carries every line and `append` holds its payload to the 16,384 bytes of the events-stay-small bound: `getConvexSize` measures the payload `{ lines, total }` at 25 bytes plus, for each line, 53 bytes and its stock item ID's UTF-8 bytes, every number counting 9 bytes whatever its value, so 100 lines at 64 bytes make 25 + 100 × (53 + 64) = 11,725 bytes and fit, while 100 lines at 128 bytes would make 18,125 bytes and at the 256 bytes a stream ID may have 30,925, which the Orders context's `append` refuses as a technical failure inside the Orders call; 64 is therefore the largest power of two at which the maximum order fits (OQ3, E-12, E-46). ## Example space From e090599ff286c13fb4806104ab7a75f554e03f03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:06:21 +0200 Subject: [PATCH 050/155] Bound PlaceOrder stock item IDs in UTF-8 bytes --- design/README.md | 2 +- example/convex/ordering.ts | 18 ++- tests/native/first-experiment-steps.ts | 147 ++++++++++++++++- ...iment.order-at-stock-item-id-bound.test.ts | 73 +++++++++ ...t.stock-item-id-past-bound-refused.test.ts | 64 ++++++++ tests/pure/stock-item-id-bound.test.ts | 152 ++++++++++++++++++ 6 files changed, 452 insertions(+), 4 deletions(-) create mode 100644 tests/native/first-experiment.order-at-stock-item-id-bound.test.ts create mode 100644 tests/native/first-experiment.stock-item-id-past-bound-refused.test.ts create mode 100644 tests/pure/stock-item-id-bound.test.ts diff --git a/design/README.md b/design/README.md index ec7b122..81bf5de 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -225 specs · 5 packs · 121 anchors → 351 nodes · 1192 edges (0 errors, 0 warnings) +225 specs · 5 packs · 123 anchors → 353 nodes · 1194 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/example/convex/ordering.ts b/example/convex/ordering.ts index 5ba847f..f349542 100644 --- a/example/convex/ordering.ts +++ b/example/convex/ordering.ts @@ -9,6 +9,7 @@ import { v, type Infer } from "convex/values"; import { internalCommand, publicCommand, + utf8Length, type CommandDeclaration, } from "../../src/command/index.js"; import { components } from "./_generated/api.js"; @@ -18,6 +19,8 @@ export const placeOrderPermission = "orders.place"; export const cancelOrderPermission = "orders.cancel"; // The largest order PlaceOrder accepts: the example's promise, chosen under the adapter's ceilings. export const maxOrderLines = 100; +// Keeps the maximum order's OrderPlaced payload under the journal's bound. +export const maxStockItemIdBytes = 64; const orderSource = { contextId: "orders", streamType: "order" }; const stockItemSource = { contextId: "inventory", streamType: "stockItem" }; const placeOrderInput = v.object({ @@ -36,8 +39,19 @@ export const placeOrderDeclaration: CommandDeclaration< name: "PlaceOrder", contractVersion: 1, input: placeOrderInput, - refine: ({ lines }) => - lines.length === 0 ? { message: "An order needs at least one line" } : null, + refine: ({ lines }) => { + if (lines.length === 0) + return { message: "An order needs at least one line" }; + for (const [line, { stockItemId }] of lines.entries()) { + const length = utf8Length(stockItemId); + if (length > maxStockItemIdBytes) + return { + message: `Line ${line} needs a stock item ID of at most ${maxStockItemIdBytes} bytes of UTF-8, not ${length}`, + details: { line, length, limit: maxStockItemIdBytes }, + }; + } + return null; + }, output: placeOrderResult, permission: { permission: placeOrderPermission }, writes: [orderSource, stockItemSource], diff --git a/tests/native/first-experiment-steps.ts b/tests/native/first-experiment-steps.ts index 7e3a338..60ad700 100644 --- a/tests/native/first-experiment-steps.ts +++ b/tests/native/first-experiment-steps.ts @@ -8,10 +8,12 @@ import { getFunctionName, type FunctionReturnType, } from "convex/server"; +import { ConvexError, type Value } from "convex/values"; import { expect } from "vitest"; import { api, internal } from "../../example/convex/_generated/api.js"; import { maxOrderLines, + maxStockItemIdBytes, placeOrderDeclaration, placeOrderPermission, } from "../../example/convex/ordering.js"; @@ -22,7 +24,7 @@ import type { Backend } from "../../harness/backend.js"; import { ordinaryClient } from "../../harness/clients.js"; import { measure, productionBackend, required } from "../../harness/native.js"; import { scheduledRows } from "./scheduled-rows.js"; -import type { MutationCtx } from "../../src/command/index.js"; +import { utf8Length, type MutationCtx } from "../../src/command/index.js"; export const tenantId = "t-1"; export const subject = "user-1"; const operator = { kind: "operator", id: "native-test" } as const; @@ -391,3 +393,146 @@ export function recordUsage(world: ExperimentWorld) { ), }); } + +export async function storedOrderDocuments(backend: Backend) { + return { + receipts: await backend.admin.readTable("receipts"), + orderStreams: await backend.admin.readTable("streams", { + component: "orders", + }), + orderEvents: await backend.admin.readTable("events", { + component: "orders", + }), + stockStreams: await backend.admin.readTable("streams", { + component: "inventory", + }), + stockEvents: await backend.admin.readTable("events", { + component: "inventory", + }), + summaries: await backend.admin.readTable("orderSummaries"), + }; +} +export interface StockItemIdWorld extends ExperimentWorld { + lines?: OrderLine[]; + response?: PlaceOrderResponse; + error?: unknown; + own?: CompletionRecord; + before?: Awaited>; +} +export function orderAtIdBound( + world: StockItemIdWorld, + size: Size, + contention: "absent" | "present", +) { + expect(size).toBe("the maximum"); + expect(contention).toBe("absent"); + world.lines = orderLines(linesOf[size], "sku"); + expect(world.lines).toHaveLength(100); +} +export async function stockItemIdBytesAre( + world: StockItemIdWorld, + idBytes: number, + lastIdBytes: number, +) { + const lines = required(world.lines, "the lines"); + for (const [i, line] of lines.entries()) + line.stockItemId = + i === lines.length - 1 + ? (lastIdBytes % 2 === 1 ? "a" : "") + + "é".repeat(Math.floor(lastIdBytes / 2)) + : line.stockItemId.padEnd(idBytes, "x"); + expect(maxStockItemIdBytes).toBe(64); + expect(new Set(lines.map(({ stockItemId }) => stockItemId)).size).toBe(100); + for (const [i, { stockItemId }] of lines.entries()) + expect(utf8Length(stockItemId)).toBe( + i === lines.length - 1 ? lastIdBytes : idBytes, + ); + const lastId = required(lines.at(-1), "the last line").stockItemId; + expect(lastId.length).toBeLessThan(utf8Length(lastId)); + await receive(world, lines); +} +export async function placeOrderAtIdBoundRuns( + world: StockItemIdWorld, + run: "the PlaceOrder use case" | "the end-to-end path", +) { + expect(run).toBe("the PlaceOrder use case"); + const backend = required(world.backend, "the backend"); + const client = required(world.client, "the client"); + world.before = await storedOrderDocuments(backend); + const mark = await backend.admin.logMark(); + await client + .mutation(api.ordering.placeOrder, { + tenantId, + requestKey: "k-order-at-id-bound", + input: { + orderId: "order-at-id-bound", + lines: required(world.lines, "the lines"), + }, + }) + .then( + (response) => { + world.response = response; + }, + (error: unknown) => { + world.error = error; + }, + ); + const records = await backend.admin.completionsSince(mark, (records) => + records.some( + (record) => + record.identifier === placeOrderIdentifier && + record.componentPath === null, + ), + ); + world.own = required( + records.find( + (record) => + record.identifier === placeOrderIdentifier && + record.componentPath === null, + ), + "the command's completion record", + ); +} +export function stockItemIdAnswerIs( + world: StockItemIdWorld, + answer: "the result" | "the rejection invalidInput", +) { + if (answer === "the result") { + expect(world.error).toBeUndefined(); + expect(world.response).toMatchObject({ + kind: "applied", + replayed: false, + result: { orderId: "order-at-id-bound", lineCount: 100, total: 14950 }, + }); + expect(required(world.own, "the completion record").error).toBeNull(); + } else { + expect(world.response).toBeUndefined(); + expect(world.error).toBeInstanceOf(ConvexError); + expect(required(world.own, "the completion record").error).toMatch( + /^Uncaught ConvexError: /, + ); + } +} +export function stockItemIdRejectionIs( + world: StockItemIdWorld, + line: number, + length: number, + limit: number, +) { + expect((world.error as ConvexError).data).toEqual({ + kind: "rejection", + code: "invalidInput", + commandType: "PlaceOrder", + message: `Line ${line} needs a stock item ID of at most ${limit} bytes of UTF-8, not ${length}`, + details: { line, length, limit }, + }); +} +export function commandDocumentsAre( + world: StockItemIdWorld, + readDocuments: number, + writtenDocuments: number, +) { + const usage = usageOf(required(world.own, "the completion record")); + expect(usage.databaseReadDocuments).toBe(readDocuments); + expect(usage.databaseWriteDocuments).toBe(writtenDocuments); +} diff --git a/tests/native/first-experiment.order-at-stock-item-id-bound.test.ts b/tests/native/first-experiment.order-at-stock-item-id-bound.test.ts new file mode 100644 index 0000000..a351779 --- /dev/null +++ b/tests/native/first-experiment.order-at-stock-item-id-bound.test.ts @@ -0,0 +1,73 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { getConvexSize } from "convex/values"; +import { expect } from "vitest"; +import { orderAtStockItemIdBoundContract as contract } from "../../generated/contracts/application.first-experiment.order-at-stock-item-id-bound.contract.js"; +import { required } from "../../harness/native.js"; +import { limitPayloadBytes } from "../../src/context/index.js"; +import { + application, + configurationIs, + orderAtIdBound, + stockItemIdBytesAre, + placeOrderAtIdBoundRuns, + stockItemIdAnswerIs, + commandDocumentsAre, + type StockItemIdWorld, +} from "./first-experiment-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.first-experiment.order-at-stock-item-id-bound", + ), + verifies: ref( + "spec:application.first-experiment.order-at-stock-item-id-bound", + ), +}); +void anchor; +bindExample(contract, (): StockItemIdWorld => ({}), { + "the production composition on a native backend": (world) => + application(world), + "an order of {size} with stock contention {contention}": ( + world, + { size, contention }, + ) => orderAtIdBound(world, size, contention), + "the order's stock item IDs are {idBytes} bytes of UTF-8, the last line's {lastIdBytes}": + (world, { idBytes, lastIdBytes }) => + stockItemIdBytesAre(world, idBytes, lastIdBytes), + "the backend runs with {configuration}": (world, { configuration }) => + configurationIs(world, configuration), + "{run} runs": (world, { run }) => placeOrderAtIdBoundRuns(world, run), + "the caller receives {answer}": (world, { answer }) => + stockItemIdAnswerIs(world, answer), + "the command read {readDocuments} documents and wrote {writtenDocuments}": ( + world, + { readDocuments, writtenDocuments }, + ) => commandDocumentsAre(world, readDocuments, writtenDocuments), + "the order's OrderPlaced payload measures {payloadBytes} bytes": async ( + world, + { payloadBytes }, + ) => { + const events = await required(world.backend, "the backend").admin.readTable( + "events", + { component: "orders" }, + ); + const own = events.filter( + (event) => event["streamId"] === "order-at-id-bound", + ); + expect(own).toHaveLength(1); + expect(own[0]).toMatchObject({ + eventType: "OrderPlaced", + payload: { lines: world.lines, total: 14950 }, + }); + const bytes = getConvexSize( + required(own[0]?.["payload"], "the OrderPlaced payload"), + ); + expect(bytes).toBe(payloadBytes); + expect(limitPayloadBytes).toBe(16384); + expect(bytes).toBeLessThanOrEqual(limitPayloadBytes); + }, +}); diff --git a/tests/native/first-experiment.stock-item-id-past-bound-refused.test.ts b/tests/native/first-experiment.stock-item-id-past-bound-refused.test.ts new file mode 100644 index 0000000..5acbe2c --- /dev/null +++ b/tests/native/first-experiment.stock-item-id-past-bound-refused.test.ts @@ -0,0 +1,64 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { stockItemIdPastBoundRefusedContract as contract } from "../../generated/contracts/application.first-experiment.stock-item-id-past-bound-refused.contract.js"; +import { required } from "../../harness/native.js"; +import { + application, + configurationIs, + orderAtIdBound, + stockItemIdBytesAre, + placeOrderAtIdBoundRuns, + stockItemIdAnswerIs, + commandDocumentsAre, + stockItemIdRejectionIs, + storedOrderDocuments, + type StockItemIdWorld, +} from "./first-experiment-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.first-experiment.stock-item-id-past-bound-refused", + ), + verifies: ref( + "spec:application.first-experiment.stock-item-id-past-bound-refused", + ), +}); +void anchor; +bindExample( + contract, + (): StockItemIdWorld => ({}), + { + "the production composition on a native backend": (world) => + application(world), + "an order of {size} with stock contention {contention}": ( + world, + { size, contention }, + ) => orderAtIdBound(world, size, contention), + "the order's stock item IDs are {idBytes} bytes of UTF-8, the last line's {lastIdBytes}": + (world, { idBytes, lastIdBytes }) => + stockItemIdBytesAre(world, idBytes, lastIdBytes), + "the backend runs with {configuration}": (world, { configuration }) => + configurationIs(world, configuration), + "{run} runs": (world, { run }) => placeOrderAtIdBoundRuns(world, run), + "the caller receives {answer}": (world, { answer }) => + stockItemIdAnswerIs(world, answer), + "the command read {readDocuments} documents and wrote {writtenDocuments}": ( + world, + { readDocuments, writtenDocuments }, + ) => commandDocumentsAre(world, readDocuments, writtenDocuments), + "the rejection names line {line} with a stock item ID of {length} bytes against a bound of {limit}": + (world, { line, length, limit }) => + stockItemIdRejectionIs(world, line, length, limit), + }, + async (world) => { + expect( + await storedOrderDocuments(required(world.backend, "the backend")), + ).toEqual( + required(world.before, "the stored documents before the command"), + ); + }, +); diff --git a/tests/pure/stock-item-id-bound.test.ts b/tests/pure/stock-item-id-bound.test.ts new file mode 100644 index 0000000..9bad149 --- /dev/null +++ b/tests/pure/stock-item-id-bound.test.ts @@ -0,0 +1,152 @@ +import { ConvexError, getConvexSize, type Value } from "convex/values"; +import { expect, test } from "vitest"; +import { + maxOrderLines, + maxStockItemIdBytes, + placeOrderDeclaration, +} from "../../example/convex/ordering.js"; +import { orderDecider, type OrderLine } from "../../example/domain/index.js"; +import { runPipeline, type MutationCtx } from "../../src/command/index.js"; +import { limitPayloadBytes } from "../../src/context/index.js"; +const linesAt = (bytes: number): OrderLine[] => + Array.from({ length: maxOrderLines }, (_, i) => ({ + stockItemId: String(i).padStart(3, "0").padEnd(bytes, "x"), + quantity: 1, + unitPrice: 0, + })); +function payloadOf(lines: OrderLine[]) { + const decision = orderDecider.decide( + orderDecider.initial(), + { commandType: "place", lines }, + { now: 0, actor: { kind: "human", id: "user" }, facts: {} }, + ); + if (decision.kind !== "applied") + throw new Error(`Expected applied, got ${decision.kind}`); + expect(decision.events).toHaveLength(1); + const event = decision.events[0]; + if (event?.eventType !== "OrderPlaced") + throw new Error("Expected OrderPlaced"); + return event.payload; +} + +test.each([ + [64, 11725, true], + [128, 18125, false], + [256, 30925, false], +] as const)( + "pure: OrderPlaced with 100 stock item IDs of %s bytes measures %s bytes", + (bytes, expected, fits) => { + const size = getConvexSize(payloadOf(linesAt(bytes))); + expect(size).toBe(expected); + expect(size).toBe(25 + 100 * (53 + bytes)); + expect(size <= limitPayloadBytes).toBe(fits); + }, +); + +test("pure: the smallest and largest safe quantities and unit prices do not change the payload size", () => { + const smallest = linesAt(64); + const largestQuantity = smallest.map((line) => ({ + ...line, + quantity: Number.MAX_SAFE_INTEGER, + })); + const largestPrice = smallest.map((line, i) => ({ + ...line, + unitPrice: i === 0 ? Number.MAX_SAFE_INTEGER : 0, + })); + for (const lines of [smallest, largestQuantity, largestPrice]) + expect(getConvexSize(payloadOf(lines))).toBe(11725); +}); + +test("pure: 64 is the largest power of two whose maximum order fits the payload bound", () => { + expect(maxStockItemIdBytes).toBe(64); + expect(maxOrderLines).toBe(100); + expect(limitPayloadBytes).toBe(16384); + let largest = 0; + for (let bytes = 1; bytes <= 256; bytes *= 2) + if (getConvexSize(payloadOf(linesAt(bytes))) <= limitPayloadBytes) + largest = bytes; + expect(maxStockItemIdBytes).toBe(largest); +}); +const refine = (ids: string[]) => { + const input = { + orderId: "order", + lines: ids.map((stockItemId) => ({ + stockItemId, + quantity: 1, + unitPrice: 0, + })), + }; + if (placeOrderDeclaration.refine === undefined) + throw new Error("PlaceOrder needs its refinement"); + return placeOrderDeclaration.refine(input); +}; +const refusal = (line: number, length: number) => ({ + message: `Line ${line} needs a stock item ID of at most 64 bytes of UTF-8, not ${length}`, + details: { line, length, limit: 64 }, +}); + +test("pure: a stock item ID at 64 bytes passes refinement", () => { + expect(refine(["a".repeat(64)])).toBeNull(); + expect(refine(["é".repeat(32)])).toBeNull(); +}); + +test("pure: a stock item ID at 65 bytes is refused with its message and details", () => { + expect(refine(["a".repeat(65)])).toEqual(refusal(0, 65)); +}); + +test("pure: refinement counts multi-byte stock item IDs in UTF-8 bytes", () => { + const id = "a" + "é".repeat(32); + expect(id.length).toBe(33); + expect(refine([id])).toEqual(refusal(0, 65)); +}); + +test("pure: refinement names the first offending line from zero", () => { + expect(refine(["a".repeat(64), "é".repeat(33), "a".repeat(65)])).toEqual( + refusal(1, 66), + ); +}); + +test("pure: an empty order keeps its own refusal", () => { + expect(refine([])).toEqual({ message: "An order needs at least one line" }); +}); + +test("pure: the pipeline refuses a long ID before any read and checks the line count first", async () => { + const touched: string[] = []; + const ctx = new Proxy({} as MutationCtx, { + get: (_target, property) => { + touched.push(String(property)); + throw new Error("The refused command reached its context"); + }, + }); + for (const count of [100, 101]) { + const lines = linesAt(64); + lines[0] = { stockItemId: "a" + "é".repeat(32), quantity: 1, unitPrice: 0 }; + if (count === 101) + lines.push({ stockItemId: "extra", quantity: 1, unitPrice: 0 }); + const error: unknown = await runPipeline(ctx, placeOrderDeclaration, { + tenantId: "tenant", + namespace: "public", + actor: { kind: "human", id: "user" }, + requestKey: "request", + input: { orderId: "order", lines }, + }).catch((error: unknown) => error); + expect(error).toBeInstanceOf(ConvexError); + expect((error as ConvexError).data).toEqual( + count === 100 + ? { + ...refusal(0, 65), + kind: "rejection", + code: "invalidInput", + commandType: "PlaceOrder", + } + : { + kind: "rejection", + code: "operationTooLarge", + commandType: "PlaceOrder", + message: "PlaceOrder takes at most 100 items, not 101", + details: { items: 101, maxItems: 100 }, + }, + ); + } + expect(touched).toEqual([]); +}); From 6c3e4ca438be6f62f087c4ebab9bc44c0a511c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:07:35 +0200 Subject: [PATCH 051/155] build: grants are held by the gate, and an operator is trimmed before its bound insertGrant and revokeGrant read the gate for all and the grant's tenant before anything else, so a closed scope or the restore door refuses a composition's grant mutation and stores nothing, the tenant row included. assertOperator measures the trimmed operator. Co-Authored-By: Claude Opus 5.5 --- src/command/actor-and-scope.ts | 9 ++- src/command/authority.ts | 23 +++++- tests/pure/write-pause.test.ts | 11 +++ tests/simulator/gate-pipeline.test.ts | 4 ++ tests/simulator/write-pause.test.ts | 100 ++++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 6 deletions(-) diff --git a/src/command/actor-and-scope.ts b/src/command/actor-and-scope.ts index 0d7c4d2..ab320ef 100644 --- a/src/command/actor-and-scope.ts +++ b/src/command/actor-and-scope.ts @@ -77,11 +77,14 @@ export type AuthorizeInput = { export const operatorValidator = v.string(); // The bound limitCallText gives an actor's id. export const limitOperatorBytes = limitActorIdLength; -// An operator entry calls this first, so a refused entry has read and written nothing. +// An operator entry calls this first, so a refused entry has read and written nothing. The operator is +// trimmed before both checks, so whitespace alone, of any length, is an empty operator; the argument is +// returned and recorded unchanged. export function assertOperator(operator: string): string { - if (operator.trim() === "") + const trimmed = operator.trim(); + if (trimmed === "") throw new Error("An operator entry needs a stated operator"); - const bytes = utf8Length(operator); + const bytes = utf8Length(trimmed); if (bytes > limitOperatorBytes) throw new Error( `The stated operator is ${bytes} bytes, above the limit of ${limitOperatorBytes}`, diff --git a/src/command/authority.ts b/src/command/authority.ts index 27df3f8..ddb9629 100644 --- a/src/command/authority.ts +++ b/src/command/authority.ts @@ -3,7 +3,9 @@ // authorizeQuery, which a parent query calls before it discloses anything, the plain helpers an // operator's function uses to write and delete a grant, and nextTenant, which reads the tenant list. // None is registered here. -import type { GenericQueryCtx } from "convex/server"; +import type { GenericDatabaseReader, GenericQueryCtx } from "convex/server"; +import { assertWritable, scopesOfUseCase } from "../gate/gate.js"; +import type { GateDataModel } from "../gate/tables.js"; import type { Actor, AuthorizeInput, SubjectRef } from "./actor-and-scope.js"; import { reject } from "./outcome-boundary.js"; import type { @@ -85,12 +87,25 @@ export type GrantInput = { subject?: SubjectRef; grantedBy: string; }; +// A grant is written like any other write: a closed "all" or tenant scope, or the restore door, +// refuses it with writePaused before anything is read. The library types db with the command tables; +// every composition that writes grants holds the gate's table too. +async function assertGrantWritable( + ctx: { db: MutationCtx["db"] }, + tenantId: string, +): Promise { + await assertWritable( + ctx as unknown as { db: GenericDatabaseReader }, + scopesOfUseCase(tenantId, []), + ); +} // Inserts the tenant's row in the tenant list when it has none, then the grant, in one mutation, so no // tenant holds a grant without a row. export async function insertGrant( ctx: { db: MutationCtx["db"] }, grant: GrantInput, ): Promise { + await assertGrantWritable(ctx, grant.tenantId); const now = Date.now(); const tenant = await ctx.db .query("tenants") @@ -118,12 +133,14 @@ export async function nextTenant( return row === null ? null : row.tenantId; } // Revocation deletes the rows that grant this permission to this principal, on this subject when one -// is named and otherwise on any; the next command's read finds none. Returns the number deleted. It -// reads past the command path's bound of limitGrantsRead, so it can reduce a principal that holds more. +// is named and otherwise on any, after the same gate read as insertGrant; the next command's read +// finds none. Returns the number deleted. It reads past the command path's bound of limitGrantsRead, +// so it can reduce a principal that holds more. export async function revokeGrant( ctx: { db: MutationCtx["db"] }, grant: Omit, ): Promise { + await assertGrantWritable(ctx, grant.tenantId); const rows = ( await ctx.db .query("grants") diff --git a/tests/pure/write-pause.test.ts b/tests/pure/write-pause.test.ts index 78c4868..00bd948 100644 --- a/tests/pure/write-pause.test.ts +++ b/tests/pure/write-pause.test.ts @@ -28,6 +28,17 @@ test("pure: assertOperator refuses an empty or whitespace-only operator and one expect(assertOperator(" ops-1 ")).toBe(" ops-1 "); }); +test("pure: assertOperator trims before both checks, so whitespace of any length is empty and the bound counts the trimmed bytes", () => { + expect(() => assertOperator(" ".repeat(600))).toThrow( + "An operator entry needs a stated operator", + ); + const padded = " ".repeat(300) + "x".repeat(512) + "\t"; + expect(assertOperator(padded)).toBe(padded); + expect(() => assertOperator(" " + "x".repeat(513) + " ")).toThrow( + "The stated operator is 513 bytes, above the limit of 512", + ); +}); + test("pure: scopesOfUseCase answers all, the tenant scope, then one source scope per stream type the declaration writes, in that order", () => { expect( scopesOfUseCase("t-1", [ diff --git a/tests/simulator/gate-pipeline.test.ts b/tests/simulator/gate-pipeline.test.ts index 53baee0..9834b76 100644 --- a/tests/simulator/gate-pipeline.test.ts +++ b/tests/simulator/gate-pipeline.test.ts @@ -146,12 +146,16 @@ test("convex-test: a duplicate in a closed tenant returns its receipt before adm await user.mutation(api.depotCommands.createDocument, call), ).toMatchObject({ replayed: true, operationId: first.operationId }); expect(await stored(t)).toEqual(before); + // A grant mutation is a writer the gate holds (spec:command.actor-and-scope fnRevokeGrant), so the + // tenant is reopened for the revoke and closed again. + await resume(t); await t.mutation(internal.grants.revoke, { tenantId: "t", principalKind: "human", principalId: `${issuer}|caller`, permission: permissions.documents, }); + await close(t); await expect( user.mutation(api.depotCommands.createDocument, call), ).rejects.toMatchObject({ data: { kind: "rejection", code: "forbidden" } }); diff --git a/tests/simulator/write-pause.test.ts b/tests/simulator/write-pause.test.ts index bd8555c..fc66b3f 100644 --- a/tests/simulator/write-pause.test.ts +++ b/tests/simulator/write-pause.test.ts @@ -20,7 +20,9 @@ import { type CommandDeclaration, type PipelineCall, } from "../../src/command/index.js"; +import { internal as productionInternal } from "../../example/convex/_generated/api.js"; import { gateAllows } from "../../src/gate/index.js"; +import { productionTest } from "./production.js"; const { version } = JSON.parse( readFileSync( join(import.meta.dirname, "../../node_modules/convex-test/package.json"), @@ -501,6 +503,104 @@ describe("the operator entries", () => { ); }); +describe("a grant under the gate", () => { + const grantArgs = (tenantId: string) => ({ + tenantId, + principalKind: "human" as const, + principalId: "user-1", + permission: "documents.write", + }); + const paused = (message: string) => ({ + kind: "transient", + code: "writePaused", + message, + }); + async function pausedData(promise: Promise) { + const error = await failure(promise); + expect(error).toBeInstanceOf(ConvexError); + return (error as ConvexError).data; + } + + test( + name( + "the fixture's grant and revoke mutations are refused with writePaused in a closed tenant and store nothing, and write in another tenant", + ), + async () => { + const t = app(); + const stored = () => + t.run(async (ctx) => ({ + grants: await ctx.db.query("grants").collect(), + tenants: await ctx.db.query("tenants").collect(), + })); + await t.mutation(internal.grants.grant, { + ...grantArgs("t-1"), + grantedBy: "operator", + }); + await close(t, "tenant:t-1", "move"); + await close(t, "tenant:t-2", "hold"); + const before = await stored(); + // A first grant in a closed tenant: neither the grant nor the tenant row is written. + expect( + await pausedData( + t.mutation(internal.grants.grant, { + ...grantArgs("t-2"), + grantedBy: "operator", + }), + ), + ).toEqual(paused("write paused for tenant:t-2: hold")); + expect( + await pausedData(t.mutation(internal.grants.revoke, grantArgs("t-1"))), + ).toEqual(paused("write paused for tenant:t-1: move")); + expect(await stored()).toEqual(before); + await t.mutation(internal.grants.grant, { + ...grantArgs("t-3"), + grantedBy: "operator", + }); + expect((await stored()).tenants).toHaveLength(2); + }, + ); + + test( + name( + "the production composition's grant and revoke mutations are refused with writePaused in a closed tenant and under the restore door, and store nothing", + ), + async () => { + const t = productionTest(); + const stored = () => + t.run(async (ctx) => ({ + grants: await ctx.db.query("grants").collect(), + tenants: await ctx.db.query("tenants").collect(), + })); + await t.mutation(productionInternal.grants.grant, { + ...grantArgs("t-1"), + grantedBy: "operator", + }); + await t.mutation(productionInternal.gate.closeGate, { + scopeKey: "tenant:t-2", + reason: "hold", + operator: "ops-1", + }); + const before = await stored(); + expect( + await pausedData( + t.mutation(productionInternal.grants.grant, { + ...grantArgs("t-2"), + grantedBy: "operator", + }), + ), + ).toEqual(paused("write paused for tenant:t-2: hold")); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + expect( + await pausedData( + t.mutation(productionInternal.grants.revoke, grantArgs("t-1")), + ), + ).toEqual(paused("write paused for all: restore")); + vi.unstubAllEnvs(); + expect(await stored()).toEqual(before); + }, + ); +}); + describe("the tenant list", () => { test( name( From eebdda61ab7fa88cbd0c5b52fae883ee16ca3401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:09:27 +0200 Subject: [PATCH 052/155] example: the comment beside the stock item ID bound says what it bounds and why Co-Authored-By: Claude Opus 5.5 --- example/convex/ordering.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/example/convex/ordering.ts b/example/convex/ordering.ts index f349542..3896c62 100644 --- a/example/convex/ordering.ts +++ b/example/convex/ordering.ts @@ -19,7 +19,8 @@ export const placeOrderPermission = "orders.place"; export const cancelOrderPermission = "orders.cancel"; // The largest order PlaceOrder accepts: the example's promise, chosen under the adapter's ceilings. export const maxOrderLines = 100; -// Keeps the maximum order's OrderPlaced payload under the journal's bound. +// The longest stock item ID an order line may carry, in UTF-8 bytes, so that the OrderPlaced of the +// largest order fits the journal's payload bound. export const maxStockItemIdBytes = 64; const orderSource = { contextId: "orders", streamType: "order" }; const stockItemSource = { contextId: "inventory", streamType: "stockItem" }; From 1afc84aa2a44220a624b4108a5361722b06a7d17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:09:59 +0200 Subject: [PATCH 053/155] design: the gate's rulings in the Specs, and the restore door's example bound Every command that reaches step 7 reads the gate, a duplicate answered from its receipt does not; the grant helpers read the gate before they write; getGateAudit answers a page of 100 to a request for more; the operator's bound counts its bytes after trimming; the restore door's switch with no deploy is an observation of the pinned release, bound by a native example; the native harness records how a lost race shows in the function log. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 7 +- design/specs/application.pack.sdp.md | 3 +- ...e-pause.restore-door-without-deploy.sdp.md | 32 ++++ design/specs/application/write-pause.sdp.md | 13 +- design/specs/command/actor-and-scope.sdp.md | 15 +- .../command/tenancy-and-authority.sdp.md | 1 + design/specs/platform/native-harness.sdp.md | 1 + ...-pause.restore-door-without-deploy.test.ts | 149 ++++++++++++++++++ tests/native/write-pause.test.ts | 59 +------ 9 files changed, 207 insertions(+), 73 deletions(-) create mode 100644 design/specs/application/write-pause.restore-door-without-deploy.sdp.md create mode 100644 tests/native/write-pause.restore-door-without-deploy.test.ts diff --git a/design/README.md b/design/README.md index a92a9f3..0d7854c 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -202 specs · 5 packs · 101 anchors → 308 nodes · 1097 edges (0 errors, 0 warnings) +203 specs · 5 packs · 102 anchors → 310 nodes · 1101 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -303,7 +303,7 @@ Slice S1 added three more for the same reason, when its scenario tests bound the Package B therefore has 7 examples, Package C has 15, and the corpus has 57. -The Layer 2 design added two more for the same reason, and nine examples that verify no doc row: +The Layer 2 design added two more for the same reason, and ten examples that verify no doc row: | Sc | Added example | Reason | |---|---|---| @@ -314,12 +314,13 @@ The Layer 2 design added two more for the same reason, and nine examples that ve | none | `spec:application.read-models.query-refused-before-disclosure` | verifies Law 5 for a parent query called with no identity | | none | `spec:application.read-models.query-refused-without-grant` | verifies Law 5 for a parent query called with no grant in the tenant | | none | `spec:application.read-models.list-pages-by-cursor` | verifies a parent list paged by a cursor pair, under E-24 | +| none | `spec:application.write-pause.restore-door-without-deploy` | shows on the pinned release that the restore door's environment switch needs no deploy, under E-42 | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-built-in-paginate-throws-in-component` | Probe 5: the built-in `paginate` throws inside a component | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split` | Probe 5: a full first page under a row cap of twice the page size is not split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap` | Probe 5: the same page under a row cap equal to the page size is split | | none | `spec:facts.f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap` | Probe 5: a parent list's page outgrows its row cap | -Package A therefore has 79 Specs, Package C has 16 examples, Package D has 19, and the corpus has 59 examples that bind a doc row. +Package A therefore has 79 Specs, Package C has 16 examples, Package D has 20, and the corpus has 59 examples that bind a doc row. ## Integration notes diff --git a/design/specs/application.pack.sdp.md b/design/specs/application.pack.sdp.md index ebf6e49..15f87eb 100644 --- a/design/specs/application.pack.sdp.md +++ b/design/specs/application.pack.sdp.md @@ -19,6 +19,7 @@ specs: - spec:application.generation-registry.first-activation-makes-read-model-writable - spec:application.generation-registry.command-without-generation-fails - spec:application.write-pause + - spec:application.write-pause.restore-door-without-deploy - spec:application.restore - spec:application.restore.restore-representative-dataset - spec:application.first-experiment @@ -46,4 +47,4 @@ modelRefs: Package D. Layer 2: parent use cases, read models and the projection contract, online rebuild with the generation registry and the write pause, restore, baseline operations, the cost-target constraints and the first experiment, with the Layer 2 scenarios and the all-layers audit scenario. -Membership order follows the plan's inventory with each parent's examples directly after it: parent use cases, read models, the projection contract, rebuild with its registry and gate, restore, the first experiment with the example domain, baseline operations, then the eight constraints. Thirty-seven Specs: nine application Specs, one operations Spec, eight constraints and nineteen examples. Three examples are siblings the plan's table did not list, `second-context-throws`, `committed-state-read-through-query` and `write-pause-rebuild-abort`, added because each doc row enumerates two cases. Five examples verify no doc row: `first-activation-makes-read-model-writable` and `command-without-generation-fails` verify the generation registry's first rebuild and the failure of a command before it under E-8, `query-refused-before-disclosure` and `query-refused-without-grant` verify Law 5 for a parent query, and `list-pages-by-cursor` verifies a parent list under E-24. +Membership order follows the plan's inventory with each parent's examples directly after it: parent use cases, read models, the projection contract, rebuild with its registry and gate, restore, the first experiment with the example domain, baseline operations, then the eight constraints. Thirty-eight Specs: nine application Specs, one operations Spec, eight constraints and twenty examples. Three examples are siblings the plan's table did not list, `second-context-throws`, `committed-state-read-through-query` and `write-pause-rebuild-abort`, added because each doc row enumerates two cases. Six examples verify no doc row: `first-activation-makes-read-model-writable` and `command-without-generation-fails` verify the generation registry's first rebuild and the failure of a command before it under E-8, `query-refused-before-disclosure` and `query-refused-without-grant` verify Law 5 for a parent query, `list-pages-by-cursor` verifies a parent list under E-24, and `restore-door-without-deploy` shows on the pinned release that the restore door's environment switch needs no deploy, under E-42. diff --git a/design/specs/application/write-pause.restore-door-without-deploy.sdp.md b/design/specs/application/write-pause.restore-door-without-deploy.sdp.md new file mode 100644 index 0000000..0fda413 --- /dev/null +++ b/design/specs/application/write-pause.restore-door-without-deploy.sdp.md @@ -0,0 +1,32 @@ +--- +id: spec:application.write-pause.restore-door-without-deploy +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.write-pause + verifies: spec:application.write-pause +--- +# The restore door closes and opens with no deploy + +E-42 · native tier · fixture composition · no acceptance row; the example shows, on the pinned backend release, that a change of the deployment's environment reaches the next function run, which the restore door rests on. + +## Intent + +- outcome: Setting `MAINTENANCE_MODE` to `restore` on a running deployment refuses the next write with no deploy, and another value lets the refused request keys apply. (E-42, F12) + +```gwt +Given a disposable backend running the fixture composition, where a caller in each of {tenants: 2} tenants holds a grant and one command has applied +When an operator sets the environment variable MAINTENANCE_MODE to {value: "restore"} with admin access and deploys nothing +Then each tenant's next command is refused with the message {message: "write paused for all: restore"} +And the internal entry is refused with the same message {internalRefused: true} +And the gate read by getGate answers restore {restore: true} +And once the variable is set to {reopen: "off"} each refused request key applies {applied: true} +``` + +## Verification — executable + +- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- The test sets the variable through the admin API's update of environment variables, with no deploy between the change and the next call, and asserts the refusal's data exactly, `{ kind: "transient", code: "writePaused", message }`, and that no receipt beyond the first command's exists while the door is closed. +- The test also sets the value `Restore` and asserts that `getGate` answers `restore` false, because only the exact value closes the door. +- The observation is of the pinned release and is read again when the pin moves. diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index e5481b2..9295b02 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -29,7 +29,7 @@ The write pause is the maintenance gate every writer in a scope reads inside its - risk: Every use case reads the gate document, one point read per command, and a gate change makes every concurrent command re-run once under optimistic concurrency (F1, E-8) - risk: This contract states no acceptable length for a write pause; before a paused rebuild starts, its length is estimated as its backfill batches times the time of one batch plus its verify batches times the time of one batch, with the batch time taken from the first paused rebuild measured on the native backend, and whether that length is the trigger for an online history rebuild is ruled with that number and not before it (D9, D18) - assumption: Mutations are serializable under optimistic concurrency, so a writer that read the gate as open cannot commit after a close that committed first (F1) -- assumption: A mutation reads a deployment's environment variable through `process.env`, and a change of the variable is seen by the next function run without a deploy; no probe of the corpus has shown the second half on a native backend (E-42, F12) +- assumption: A mutation reads a deployment's environment variable through `process.env`, and a change of the variable is seen by the next function run without a deploy, which `observedRestoreDoorWithoutDeploy` shows on the pinned local backend and no run has shown on a hosted deployment (E-42, F12) ### Open questions @@ -37,7 +37,7 @@ The write pause is the maintenance gate every writer in a scope reads inside its ## Contract -- There is one gate document per deployment, read by the command pipeline at its step 7 for every command through `assertWritable`, which throws, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states (D9, E-8, E-25, E-42) +- There is one gate document per deployment, read through `assertWritable`, which throws, by the command pipeline at its step 7 and by the grant helpers `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` before they write, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states; every command that reaches step 7 reads the gate; a duplicate answered from its receipt does not, because its answer writes nothing (D9, D6, E-8, E-25, E-37, E-42) - A scope is `all`, one tenant, or one source stream type of one context; the gate holds the set of closed scopes, each with a reason, the generation it serves if any, and the operator who closed it (E-8, E-48, D9) - A writer in a closed scope refuses before any write with the transient refusal `writePaused`, whose message carries the scope and the reason; nothing is stored, no operation ID is minted, and the same request key succeeds once the scope is open (D9, D4, Law 6) - A use case is in scope when `all` is closed, when its tenant is closed, or when a stream type its declaration's `writes` names is closed (E-8, E-38, D9) @@ -64,7 +64,7 @@ The gate is one document so that a command pays one point read. Scopes are strin - indexGatesByKeyUse: the one point read every writer makes, `.withIndex("by_key", (q) => q.eq("key", "gates")).unique()`; an absent document reads as no closed scope (E-8, D9) - typeScopeKey: `type ScopeKey = "all" | string` where a tenant scope is `"tenant:" + tenantId` and a source scope is `"source:" + contextId + ":" + streamType`, built only by `scopesOfUseCase`, `startGeneration`, the rebuild's batches, the journal's baseline driver and the operator entries (E-8, E-38, D11) - fnGateAllows: `gateAllows(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[], allowGeneration?: Id<"generations">): Promise<{ allowed: true } | { allowed: false; scopeKey: ScopeKey; reason: string }>` the one gate read: it first reads `process.env.MAINTENANCE_MODE` and answers `{ allowed: false, scopeKey: "all", reason: "restore" }` without reading the document when the value is `restore`; otherwise it reads the gate document once and answers not allowed with the `scopeKey` and `reason` of the first entry of `closed`, in array order, whose `scopeKey` is one of `scopes` and which is not exempt, and allowed when there is none; an entry is exempt only when `allowGeneration` is given, the entry has a `generationId`, and the two are equal; it writes nothing (E-8, E-42, D9, F9) -- fnAssertWritable: `assertWritable(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[]): Promise` calling `gateAllows` with no generation and, when the answer is not allowed, throwing through the boundary's `refuseTransient({ code: "writePaused", message: "write paused for " + scopeKey + ": " + reason })`; the command pipeline's step 7 calls it before it mints the operation, and nothing else does (E-8, D9, E-35) +- fnAssertWritable: `assertWritable(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[]): Promise` calling `gateAllows` with no generation and, when the answer is not allowed, throwing through the boundary's `refuseTransient({ code: "writePaused", message: "write paused for " + scopeKey + ": " + reason })`; the command pipeline's step 7 calls it before it mints the operation, and `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` call it with `scopesOfUseCase(grant.tenantId, [])` before their first read; nothing else does (E-8, D9, E-35, E-37) - errorCodeWritePaused: `writePaused`, the transient code the outcome boundary reserves; the refusal's data is exactly `{ kind: "transient", code: "writePaused", message }` with no `retryAfterMs`, where `message` is `"write paused for all: restore"` under the restore door and `"write paused for tenant:" + tenantId + ": " + reason` for a closed tenant; a transient refusal, never a stored rejection, so no receipt, event, read-model row or audit record exists for the refused command (D4, Law 6, E-35) - restoreDoor: `gateAllows` runs in the parent and reads `process.env.MAINTENANCE_MODE` before the gate document; only the exact value `restore` closes the door, so the restore procedure keeps writers out even though the snapshot replaced the gate, and every rebuild batch parks until the accepted branch resumes it; the restore procedure's own six mutations call neither `gateAllows` nor `assertWritable`; no component reads the variable (E-42, D19, F11) - fnScopesOfUseCase: `scopesOfUseCase(tenantId: string, writes: readonly SourceRef[]): ScopeKey[]` returning `"all"`, the tenant scope and one source scope per `SourceRef` of the declaration's `writes`, in that order, so a command is held by a closed deployment, by its closed tenant and by a paused rebuild of any view whose sources it writes (E-8, E-38, D9) @@ -73,15 +73,16 @@ The gate is one document so that a command pays one point read. Scopes are strin - fnCloseGate: `export const closeGate = internalMutation({ args: { scopeKey: v.string(), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `closeScope` with no generation: it calls `assertOperator`, throws `"closeGate takes the scope all or a tenant scope, and " + scopeKey + " is neither"` unless `scopeKey` is `all` or `tenant:` followed by a tenant ID of 1 to `limitIdLength` bytes, throws `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and then calls `closeScope`; it does not read the tenant list, so a tenant may be closed before its first grant; this is also how a meaning migration's sweep is run under a pause (E-8, E-25, E-48, D9, D19) - fnResumeGate: `export const resumeGate = internalMutation({ args: { scopeKey: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `resumeScope`: it calls `assertOperator`, makes the scope check of `closeGate` with `resumeGate` in the message, so a source scope is never reopened ahead of its rebuild, and then calls `resumeScope` (E-8, E-48, D9, Sc L2-6) - fnGetGate: `export const getGate = internalQuery({ args: {}, returns: v.object({ restore: v.boolean(), closed: v.array(closedEntryValidator) }), handler })` answering `restore` true when `MAINTENANCE_MODE` is `restore` and `closed` as the document holds it, empty when there is no document (E-8, E-42, D19) -- fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page, where `operatorAuditDoc` is the `operatorAudit` table's document validator with `_id` and `_creationTime` (E-8, E-43, D19) +- fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page, a request for more answered with a page of 100, where `operatorAuditDoc` is the `operatorAudit` table's document validator with `_id` and `_creationTime` (E-8, E-43, D19) - occSerialization: a writer's mutation reads the gate document, or the index range that holds none; a close that commits first invalidates that read, the writer re-runs, reads the closed gate and refuses; a writer that commits first wrote before the close, which is the consistent cut; a batch scheduled before the close but started after it reads the closed gate like any writer, so no closing state, drain interval or confirmation is needed, and the one-second mutation timeout plays no part (F1, D9, E-8) +- observedRestoreDoorWithoutDeploy: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, a change of `MAINTENANCE_MODE` through the admin API's update of environment variables is seen by the next function run with no deploy: set to `restore`, the next command in every tenant, through the public and the internal entry, is refused with `"write paused for all: restore"` and `getGate` answers `restore` true; set to another value, `Restore` among them, the refused request keys apply; `spec:application.write-pause.restore-door-without-deploy` binds it at the native tier, and it is read again when the pin moves (E-42, F12) - limitClosedScopes: at most 16 scopes closed at once, so the gate document stays a few kilobytes and the writer check scans an array in memory (E-8, F13) - limitGateReason: a reason is 1 to 256 bytes of UTF-8, so the refusal's message stays under a kilobyte (E-8, F13) -- limitGateReadsPerCommand: 1 point read per command, and none while the restore door is closed; the first experiment counts it (E-8, Sc L2-3) +- limitGateReadsPerCommand: 1 point read per command that reaches step 7, none for a duplicate answered from its receipt, and none while the restore door is closed; the first experiment counts it (E-8, Sc L2-3) ## Verification — reviewed -- A reviewer confirms that `assertWritable` is called by the command pipeline's step 7 for every command, that `gateAllows` is called by every background batch before its first write and that the batch parks rather than throws on a refusal, that the two are one read because `assertWritable` is `gateAllows` plus the throw, that the gate's entries and the restore procedure's six mutations are the only exemptions, and that no function caches the gate across mutations. +- A reviewer confirms that `assertWritable` is called by the command pipeline's step 7 for every command that reaches it and by the two grant helpers before they write, that `gateAllows` is called by every background batch before its first write and that the batch parks rather than throws on a refusal, that the two are one read because `assertWritable` is `gateAllows` plus the throw, that the gate's entries and the restore procedure's six mutations are the only exemptions, and that no function caches the gate across mutations. - A reviewer confirms that `gateAllows` reads the environment before the document, that an entry with no generation is never exempt, and that a gate read with no generation passes no closed entry. - A reviewer confirms that no function other than `startGeneration` closes a source scope, so every closed entry that names a generation names a row that exists, and that the journal's baseline driver reads the gate with the source scope of the stream type it migrates before every batch. - A reviewer confirms that `closeScope` and `resumeScope` are the only writers of the gate document, that each inserts its audit record in the same mutation with no catch around it, and that a test whose audit table refuses the record finds the gate unchanged. diff --git a/design/specs/command/actor-and-scope.sdp.md b/design/specs/command/actor-and-scope.sdp.md index f4a5bc0..5f49fc3 100644 --- a/design/specs/command/actor-and-scope.sdp.md +++ b/design/specs/command/actor-and-scope.sdp.md @@ -16,7 +16,7 @@ relations: --- # Actor, tenant scope, namespace and grants -Layer 1 · Detail: full · Traces: D11, D2, D6, D13, Law 11, F11, E-6, E-54, E-48, E-49. +Layer 1 · Detail: full · Traces: D11, D2, D6, D13, Law 11, F11, E-6, E-8, E-54, E-48, E-49. This contract pins the shapes the parent establishes and passes into components: the `Actor` with its kind, identity and delegation reference, the `TenantScope`, the `CallerNamespace` and the `Authority` a worker runs under. It pins the `authorize` helper, the `authorizeQuery` helper a parent query calls, and the `grants` table. The event envelope's `actor` field, the receipts table's `namespace` field and the obligation record's `authority` field use these validators, so an actor and an authority are each defined once and referenced everywhere. @@ -47,10 +47,11 @@ This contract pins the shapes the parent establishes and passes into components: - A context operation receives `tenantId`, `actor` and `operation`, the `OperationRef` of `spec:context.event-envelope`, as arguments on every call, a context query receives `tenantId`, and neither reads `ctx.auth` (D11, F11) - The `actorValidator` here is the one the event envelope stores in its `actor` field and the receipts table stores as `actorId`, so an actor is defined once (D2, D6) - [extension] An operator entry is an `internalMutation` or `internalQuery`, so only admin access reaches it; it reads no grant, calls neither `establishActor` nor `authorize`, and reads no `ctx.auth` (E-48, D11, D19) -- [extension] An operator entry that changes anything takes `operator`, a required string of 1 to 512 bytes of UTF-8 that is not only whitespace, checks it with `assertOperator` before any read or write, and records it unchanged on what it changes: the generation row, the gate's closed entry and its audit record, or the restore's run record; an operator entry that only reads takes none (E-48, D19) +- [extension] An operator entry that changes anything takes `operator`, a required string that is not only whitespace and is 1 to 512 bytes of UTF-8 after trimming, checks it with `assertOperator` before any read or write, and records it unchanged on what it changes: the generation row, the gate's closed entry and its audit record, or the restore's run record; an operator entry that only reads takes none (E-48, D19) - [extension] A stated operator is what admin access chose to state: no function establishes an actor of kind `operator` from a credential, no permission names an operator duty, and the kind `operator` of `ActorKind` is carried only by an actor a trusted internal caller states (E-48, D11) - [extension] The tenant list is the `tenants` table: `insertGrant` inserts a tenant's row when it finds none, in the same mutation as the grant, so every tenant that a command or a query was ever authorized in has a row; the library never deletes a row, and revoking every grant of a tenant leaves its row (E-49, D11, Law 11) - [extension] Grant rows are inserted only through `insertGrant`, so no tenant can hold a grant without a row in the tenant list (E-49, E-37, D11) +- [extension] A grant is written and revoked only while the gate allows its tenant: `insertGrant` and `revokeGrant` read the maintenance gate for `all` and the grant's tenant before anything else, so a closed scope or the restore door refuses the composition's grant mutations with the transient `writePaused` and nothing is stored, the tenant row of a first grant included (E-49, E-37, E-8, D11) - [extension] A pass over the whole deployment, a rebuild's batches, a restore's checks and a receipt sweep among them, takes its tenants from `nextTenant`, one tenant per read in ascending tenant-ID order, and names that tenant on every read and write of tenant data; a tenant whose row is inserted behind a pass's cursor is not visited by that pass (E-49, Law 11, D11) ## Design @@ -73,9 +74,9 @@ This contract pins the shapes the parent establishes and passes into components: - fnEstablishActor: `establishActor(ctx: MutationCtx | QueryCtx, serviceIssuers: ReadonlySet): Promise` returns `null` when `ctx.auth.getUserIdentity()` is `null`, a `service` actor when the identity's issuer is in `serviceIssuers`, and a `human` actor otherwise, with `id` from `tokenIdentifier` (D11, E-37) - typeQueryPolicy: `type QueryPolicy = { name: string; tenantId: string; permission: string; subject?: SubjectRef }` what a parent query states before it discloses anything: its own name, the tenant it reads and the permission the read requires (D11, Law 5, E-6) - fnAuthorizeQuery: `authorizeQuery(ctx: GenericQueryCtx, policy: QueryPolicy, serviceIssuers?: ReadonlySet): Promise` calls `establishActor` and then `authorize` for the policy's tenant, permission and subject, as the pipeline's steps 2 and 4 do, and returns the actor; it throws through `reject` with code `unauthenticated` when there is no identity and `forbidden` with `details.reason` when the decision denies, each with the query's `name` where a command's type goes; it writes nothing, and it is generic over the parent's data model because a database reader typed with the parent's tables is not assignable to one typed with the library's (D11, Law 5, D7, E-6) -- fnInsertGrant: `insertGrant(ctx: { db: MutationCtx["db"] }, grant: GrantInput): Promise>` reads `tenants` by `by_tenant` for `grant.tenantId` and inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, then inserts one grant row with `grantedAt` the mutation's `Date.now()`, where `GrantInput = { tenantId: string; principalKind: ActorKind; principalId: string; permission: string; subject?: SubjectRef; grantedBy: string }`; a plain helper the library exports and registers no function for (D11, E-37, E-49) -- fnRevokeGrant: `revokeGrant(ctx: { db: MutationCtx["db"] }, grant: Omit): Promise` deletes the rows that grant this permission to this principal, on the named subject when one is given and otherwise on any, and returns how many it deleted; revocation is that delete (D11, E-37) -- grantWriters: the application registers its own internal mutations over `insertGrant` and `revokeGrant`; a tenant's first grant is created with admin access through that internal mutation, and later grants the same way or by whatever command the composition registers; the library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation, which are an adopter's to build on the helpers; a native test creates its grants as setup with the harness's admin access (D11, E-37, E-15) +- fnInsertGrant: `insertGrant(ctx: { db: MutationCtx["db"] }, grant: GrantInput): Promise>` first calls `assertWritable(ctx, scopesOfUseCase(grant.tenantId, []))` of `spec:application.write-pause`, so a closed `all` or tenant scope, or the restore door, refuses it with `writePaused` before it reads anything, then reads `tenants` by `by_tenant` for `grant.tenantId` and inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, then inserts one grant row with `grantedAt` the mutation's `Date.now()`, where `GrantInput = { tenantId: string; principalKind: ActorKind; principalId: string; permission: string; subject?: SubjectRef; grantedBy: string }`; a plain helper the library exports and registers no function for (D11, E-37, E-49, E-8) +- fnRevokeGrant: `revokeGrant(ctx: { db: MutationCtx["db"] }, grant: Omit): Promise` makes the gate read of `insertGrant` first, then deletes the rows that grant this permission to this principal, on the named subject when one is given and otherwise on any, and returns how many it deleted; revocation is that delete (D11, E-37, E-8) +- grantWriters: the application registers its own internal mutations over `insertGrant` and `revokeGrant`; a tenant's first grant is created with admin access through that internal mutation, and later grants the same way or by whatever command the composition registers; the library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation, which are an adopter's to build on the helpers; both helpers read the maintenance gate first, so the composition's grant mutations are held by a closed tenant, a closed deployment and the restore door like any writer; a native test creates its grants as setup with the harness's admin access (D11, E-37, E-15) - tableGrants: `grants: defineTable({ tenantId: v.string(), principalKind: actorKindValidator, principalId: v.string(), permission: v.string(), subject: v.optional(v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() })), grantedBy: v.string(), grantedAt: v.number() }).index("by_principal", ["tenantId", "principalKind", "principalId"]).index("by_permission", ["tenantId", "permission"])` (D11, Law 11, E-6) - indexGrantsByPrincipal: `.index("by_principal", ["tenantId", "principalKind", "principalId"])` (D11) - indexGrantsByPrincipalUse: the authorization read of every command and disclosing query, three equalities and `.take(limit + 1)` against the bound of `limitGrantsRead` (D11, Law 5) @@ -87,8 +88,8 @@ This contract pins the shapes the parent establishes and passes into components: - indexTenantsByTenantUse: the point read of `insertGrant` and the range read of `nextTenant`; no command and no query reads the table (E-49) - fnNextTenant: `nextTenant(ctx: { db: QueryCtx["db"] }, after: string | null): Promise` making one indexed read, `ctx.db.query("tenants").withIndex("by_tenant", (q) => (after === null ? q : q.gt("tenantId", after))).first()`, and answering that row's `tenantId`, or `null` when no tenant follows `after`; `null` as `after` asks for the first tenant (E-49, Law 11) - validatorOperator: `const operatorValidator = v.string()` the `operator` argument of every operator entry that changes anything and the type of every field that records it (E-48) -- fnAssertOperator: `assertOperator(operator: string): string` returning its argument, and throwing the plain `Error` `"An operator entry needs a stated operator"` when `operator.trim()` is empty and `"The stated operator is " + bytes + " bytes, above the limit of 512"` when `utf8Length(operator)` is above `limitOperatorBytes`; an operator entry calls it first, so a refused entry has read and written nothing (E-48, D19) -- limitOperatorBytes: 512 bytes of UTF-8, the bound `limitCallText` of `spec:command.command-pipeline` gives an actor's `id` (E-48, E-34) +- fnAssertOperator: `assertOperator(operator: string): string` returning its argument, and throwing the plain `Error` `"An operator entry needs a stated operator"` when `operator.trim()` is empty, so a whitespace-only operator of any length is refused as empty, and `"The stated operator is " + bytes + " bytes, above the limit of 512"` when `utf8Length(operator.trim())`, the operator's length after trimming, is above `limitOperatorBytes`; an operator entry calls it first, so a refused entry has read and written nothing (E-48, D19) +- limitOperatorBytes: 512 bytes of UTF-8 after trimming, the bound `limitCallText` of `spec:command.command-pipeline` gives an actor's `id` (E-48, E-34) - operatorRecordedWhere: `startedBy` and `changedBy` on the generation row of `spec:application.generation-registry`, `changedBy` on a closed entry of the gate and `operator` on its audit record in `spec:application.write-pause` and `spec:operations.baseline-operations`, and `operator` and `finishedBy` on the run record of `spec:application.restore` (E-48, D19) ## Verification — reviewed diff --git a/design/specs/command/tenancy-and-authority.sdp.md b/design/specs/command/tenancy-and-authority.sdp.md index 78a743b..1ce4969 100644 --- a/design/specs/command/tenancy-and-authority.sdp.md +++ b/design/specs/command/tenancy-and-authority.sdp.md @@ -52,6 +52,7 @@ This Spec owns the rules of tenant scope, actor establishment, authorization and - rule: Namespaces are server-assigned; the public entry assigns `public`, the internal entry accepts a namespace only from trusted server code, and a public caller cannot choose a system namespace (D6, D11, Sc L1-7) - rule: Receipt keys and local IDs are scoped by tenant, so two tenants with the same request key and local ID never collide and neither sees the other's outcome (D11, D6, Sc L1-6) - rule: A reference that crosses a context boundary is `(tenantId, contextId, eventId)` or an application ID, never a private document ID, so no ID leaks a tenant's row (D2) +- rule: [extension] A composition's grant and revoke mutations are writers the maintenance gate holds: the grant helpers read the gate for the whole deployment and the grant's tenant before they write, so a closed tenant, a closed deployment or the restore door refuses them with the transient `writePaused` of `spec:application.write-pause` and nothing is stored, the tenant row of a first grant included (E-37, E-49, E-8, D11) - rule: [extension] The public entry derives the actor from the authenticated identity's `tokenIdentifier`, `subject` and `issuer`; a revoked grant fails the next command because grants are read fresh in every transaction (E-37) - flow: The public entry reads `ctx.auth.getUserIdentity()` once; `null` throws rejection `unauthenticated`; an identity becomes an actor of kind `human`, or `service` when its issuer is configured as a service issuer (D11, E-37) - flow: The internal entry receives `actor` and `namespace` from its trusted caller, an HTTP action, a worker, a workflow step or an agent runner, which established them from its own provenance (D11, D13) diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 68ea5b7..8983bc1 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -94,6 +94,7 @@ The harness is plain test support with no Convex function in it. The entries who - observedWriteRate: [extension] the backend holds and then refuses writes past a rate, with `TooManyWrites` and the text "limited to 4 MiB bytes written per 1 second"; one transaction that writes 8.8 MiB commits, so the rate is not a bound on a transaction; this is why a test that writes more than the rate paces its transactions, and why a throughput measured on this backend states the limit (E-15, S13) - observedFunctionLog: [extension] the backend keeps its last 1000 function log entries in memory and loses them at a restart, and it logs one completion record per top-level call and none for a nested or component call inside it; the harness reads the log by polling `/api/stream_function_logs` from a mark built on the machine clock (E-15) - observedOccRetry: [extension] a completion record of a mutation attempt that failed its commit on an optimistic-concurrency conflict carries `willRetry` true and an `occInfo` naming a table, and the rerun that follows in the same request has its own record; which table `occInfo` names varied between runs of one race, the receipts, events or streams table; the harness type of a completion record leaves these fields out and the tests read them from the record (E-15) +- observedOccRerunCount: [extension] the completion record of a mutation attempt that lost an optimistic-concurrency race carries no error, `error` null, under the same request ID as the rerun that follows, so a count of completion records is a count of executions, and the reruns of one request are its completion records minus one (E-15) - observedReadCount: [extension] a completion record's `usageStats.databaseReadDocuments` counts the documents the call read, and a call refused before its handler, by argument validation or by Convex's visibility rule, has a record that read none (E-15) - observedUsageStats: [extension] a top-level mutation's completion record carries `usageStats` with `databaseReadDocuments`, `databaseWriteDocuments`, `databaseReadBytes`, `databaseWriteBytes` and `databaseWriteIndexRows`, and they count what a component call inside the mutation read and wrote (E-15) - observedFailedMutationWrites: [extension] a failed top-level mutation's completion record shows no written documents, `usageStats.databaseWriteDocuments` 0, even when a component call inside it wrote before the throw, so the function log cannot show writes that rolled back (E-15) diff --git a/tests/native/write-pause.restore-door-without-deploy.test.ts b/tests/native/write-pause.restore-door-without-deploy.test.ts new file mode 100644 index 0000000..c22f4f5 --- /dev/null +++ b/tests/native/write-pause.restore-door-without-deploy.test.ts @@ -0,0 +1,149 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { getFunctionName } from "convex/server"; +import { ConvexError, type Value } from "convex/values"; +import { expect } from "vitest"; +import { restoreDoorWithoutDeployContract as contract } from "../../generated/contracts/application.write-pause.restore-door-without-deploy.contract.js"; +import { api, internal } from "../../fixture/convex/_generated/api.js"; +import { permissions } from "../../fixture/convex/depotCommands.js"; +import type { Backend } from "../../harness/backend.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend, required } from "../../harness/native.js"; +const anchor = specTest({ + id: testAnchorId("test:application.write-pause.restore-door-without-deploy"), + verifies: ref("spec:application.write-pause.restore-door-without-deploy"), +}); +void anchor; +// The restore door is the deployment's MAINTENANCE_MODE, changed through the admin API with no deploy +// between the change and the next call. The fixture's CreateDocument is the write. +type Client = ReturnType; +interface World { + backend?: Backend; + callers?: { tenantId: string; client: Client }[]; + receiptsBefore?: number; +} +const create = (client: Client, tenantId: string, key: string) => + client.mutation(api.depotCommands.createDocument, { + tenantId, + requestKey: key, + input: { documentId: key, title: "Report" }, + }); +const internalCreate = (backend: Backend) => + backend.admin.run( + getFunctionName(internal.depotCommands.createDocumentInternal), + { + tenantId: "t-1", + namespace: "worker", + actor: { kind: "service", id: "svc-1" }, + requestKey: "internal-1", + input: { documentId: "internal-1", title: "Report" }, + }, + ); +async function thrownData(promise: Promise): Promise { + const error = await promise.then( + () => { + throw new Error("The call did not fail"); + }, + (thrown: unknown) => thrown, + ); + expect(error).toBeInstanceOf(ConvexError); + return (error as ConvexError).data; +} +const paused = (message: string) => ({ + kind: "transient", + code: "writePaused", + message, +}); +const getGate = (backend: Backend) => + backend.admin.run(getFunctionName(internal.gate.getGate), {}); +bindExample(contract, (): World => ({}), { + "a disposable backend running the fixture composition, where a caller in each of {tenants} tenants holds a grant and one command has applied": + async (world, { tenants }) => { + const backend = await fixtureBackend(); + world.backend = backend; + const grant = ( + tenantId: string, + principalKind: "human" | "service", + principalId: string, + ) => + backend.admin.run(getFunctionName(internal.grants.grant), { + tenantId, + principalKind, + principalId, + permission: permissions.documents, + grantedBy: "native-test", + }); + world.callers = []; + for (let n = 1; n <= tenants; n += 1) { + const tenantId = `t-${n}`; + await grant(tenantId, "human", `${backend.issuer.issuer}|user-${n}`); + world.callers.push({ + tenantId, + client: ordinaryClient(backend.url, { + token: await backend.issuer.token(`user-${n}`), + }), + }); + } + // Authorization, step 4, precedes the gate, step 7: the internal entry's actor holds a grant. + await grant("t-1", "service", "svc-1"); + const [first] = world.callers; + expect( + await create(required(first, "a caller").client, "t-1", "before"), + ).toMatchObject({ kind: "applied" }); + world.receiptsBefore = (await backend.admin.readTable("receipts")).length; + }, + "an operator sets the environment variable MAINTENANCE_MODE to {value} with admin access and deploys nothing": + async (world, { value }) => { + await required(world.backend, "the backend").admin.setEnvironment({ + MAINTENANCE_MODE: value, + }); + }, + "each tenant's next command is refused with the message {message}": async ( + world, + { message }, + ) => { + for (const { tenantId, client } of required(world.callers, "the callers")) + expect(await thrownData(create(client, tenantId, "door"))).toEqual( + paused(message), + ); + }, + "the internal entry is refused with the same message {internalRefused}": + async (world, { internalRefused }) => { + expect(internalRefused).toBe(true); + expect( + await thrownData(internalCreate(required(world.backend, "backend"))), + ).toEqual(paused("write paused for all: restore")); + }, + "the gate read by getGate answers restore {restore}": async ( + world, + { restore }, + ) => { + expect(await getGate(required(world.backend, "the backend"))).toEqual({ + restore, + closed: [], + }); + }, + "once the variable is set to {reopen} each refused request key applies {applied}": + async (world, { reopen, applied }) => { + const backend = required(world.backend, "the backend"); + expect(applied).toBe(true); + // Nothing of the refused calls was stored while the door was closed. + expect(await backend.admin.readTable("receipts")).toHaveLength( + required(world.receiptsBefore, "the receipts before"), + ); + // Only the exact value closes the door. + await backend.admin.setEnvironment({ MAINTENANCE_MODE: "Restore" }); + expect(await getGate(backend)).toEqual({ restore: false, closed: [] }); + await backend.admin.setEnvironment({ MAINTENANCE_MODE: reopen }); + for (const { tenantId, client } of required(world.callers, "callers")) + expect(await create(client, tenantId, "door")).toMatchObject({ + kind: "applied", + replayed: false, + }); + expect(await internalCreate(backend)).toMatchObject({ kind: "applied" }); + }, +}); diff --git a/tests/native/write-pause.test.ts b/tests/native/write-pause.test.ts index 37161e5..2ad11c7 100644 --- a/tests/native/write-pause.test.ts +++ b/tests/native/write-pause.test.ts @@ -1,7 +1,8 @@ // The maintenance gate on the native backend, over the fixture composition: a closed tenant refuses a -// public command and leaves another tenant writable, the restore door set through the deployment's -// environment refuses the next write with no deploy, a close races commands in flight, and every +// public command and leaves another tenant writable, a close races commands in flight, and every // operator action has its audit record (spec:application.write-pause, spec:operations.baseline-operations). +// The restore door's switch with no deploy is bound to its example in +// write-pause.restore-door-without-deploy.test.ts. import { getFunctionName } from "convex/server"; import { ConvexError, type Value } from "convex/values"; import { expect, test } from "vitest"; @@ -101,60 +102,6 @@ test("native: a closed tenant refuses a public command with writePaused and stor expect(await gateAudit(backend, "all")).toEqual([]); }); -test("native: MAINTENANCE_MODE set to restore on the running backend refuses the next write of every tenant and entry with no deploy, and any other value lets the same request key apply", async () => { - const backend = await fixtureBackend(); - const { admin } = backend; - const first = await userIn(backend, "t-1", "user-1"); - const second = await userIn(backend, "t-2", "user-2"); - expect(await create(first, "t-1", "doc-0", "k-0")).toMatchObject({ - kind: "applied", - }); - // Authorization, step 4, precedes the gate, step 7: the service actor holds a grant. - await admin.run(getFunctionName(internal.grants.grant), { - tenantId: "t-2", - principalKind: "service", - principalId: "svc-1", - permission: permissions.documents, - grantedBy: "native-test", - }); - await admin.setEnvironment({ MAINTENANCE_MODE: "restore" }); - const door = paused("write paused for all: restore"); - expect(await thrownData(create(first, "t-1", "doc-1", "k-1"))).toEqual(door); - expect(await thrownData(create(second, "t-2", "doc-1", "k-1"))).toEqual(door); - expect( - await thrownData( - admin.run( - getFunctionName(internal.depotCommands.createDocumentInternal), - { - tenantId: "t-2", - namespace: "worker", - actor: { kind: "service", id: "svc-1" }, - requestKey: "k-2", - input: { documentId: "doc-2", title: "Report" }, - }, - ), - ), - ).toEqual(door); - expect(await admin.run(getFunctionName(internal.gate.getGate), {})).toEqual({ - restore: true, - closed: [], - }); - expect(await admin.readTable("receipts")).toHaveLength(1); - for (const value of ["Restore", "off"]) { - await admin.setEnvironment({ MAINTENANCE_MODE: value }); - expect(await admin.run(getFunctionName(internal.gate.getGate), {})).toEqual( - { restore: false, closed: [] }, - ); - } - expect(await create(first, "t-1", "doc-1", "k-1")).toMatchObject({ - kind: "applied", - replayed: false, - }); - expect(await create(second, "t-2", "doc-1", "k-1")).toMatchObject({ - kind: "applied", - }); -}); - // Optimistic concurrency makes the close a consistent cut: a command either commits before the close // or reads the closed gate and is refused. What a client sees: every command ends applied or refused // with writePaused, never with a technical failure; an applied command left its receipt and its event From dcd72a82d8fa6a5c16e84815625b1eb31de93fdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:13:38 +0200 Subject: [PATCH 054/155] build: the trimmed text is the operator, checked and recorded assertOperator returns the trimmed operator, and closeGate and resumeGate record what it returns, so no stored operator exceeds 512 bytes. Co-Authored-By: Claude Opus 5.5 --- design/specs/command/actor-and-scope.sdp.md | 6 ++--- src/command/actor-and-scope.ts | 8 +++--- src/gate/entries.ts | 10 ++++---- tests/pure/gate.test.ts | 6 ++--- tests/pure/write-pause.test.ts | 8 +++--- tests/simulator/gate.test.ts | 12 ++++----- tests/simulator/write-pause.test.ts | 28 +++++++++++++++++++++ 7 files changed, 53 insertions(+), 25 deletions(-) diff --git a/design/specs/command/actor-and-scope.sdp.md b/design/specs/command/actor-and-scope.sdp.md index 5f49fc3..ef4f9ce 100644 --- a/design/specs/command/actor-and-scope.sdp.md +++ b/design/specs/command/actor-and-scope.sdp.md @@ -47,7 +47,7 @@ This contract pins the shapes the parent establishes and passes into components: - A context operation receives `tenantId`, `actor` and `operation`, the `OperationRef` of `spec:context.event-envelope`, as arguments on every call, a context query receives `tenantId`, and neither reads `ctx.auth` (D11, F11) - The `actorValidator` here is the one the event envelope stores in its `actor` field and the receipts table stores as `actorId`, so an actor is defined once (D2, D6) - [extension] An operator entry is an `internalMutation` or `internalQuery`, so only admin access reaches it; it reads no grant, calls neither `establishActor` nor `authorize`, and reads no `ctx.auth` (E-48, D11, D19) -- [extension] An operator entry that changes anything takes `operator`, a required string that is not only whitespace and is 1 to 512 bytes of UTF-8 after trimming, checks it with `assertOperator` before any read or write, and records it unchanged on what it changes: the generation row, the gate's closed entry and its audit record, or the restore's run record; an operator entry that only reads takes none (E-48, D19) +- [extension] An operator entry that changes anything takes `operator`, a required string that is not only whitespace and is 1 to 512 bytes of UTF-8 after trimming, checks it with `assertOperator` before any read or write, and records the trimmed text `assertOperator` returns on what it changes: the generation row, the gate's closed entry and its audit record, or the restore's run record, so no stored operator exceeds 512 bytes; an operator entry that only reads takes none (E-48, D19) - [extension] A stated operator is what admin access chose to state: no function establishes an actor of kind `operator` from a credential, no permission names an operator duty, and the kind `operator` of `ActorKind` is carried only by an actor a trusted internal caller states (E-48, D11) - [extension] The tenant list is the `tenants` table: `insertGrant` inserts a tenant's row when it finds none, in the same mutation as the grant, so every tenant that a command or a query was ever authorized in has a row; the library never deletes a row, and revoking every grant of a tenant leaves its row (E-49, D11, Law 11) - [extension] Grant rows are inserted only through `insertGrant`, so no tenant can hold a grant without a row in the tenant list (E-49, E-37, D11) @@ -88,8 +88,8 @@ This contract pins the shapes the parent establishes and passes into components: - indexTenantsByTenantUse: the point read of `insertGrant` and the range read of `nextTenant`; no command and no query reads the table (E-49) - fnNextTenant: `nextTenant(ctx: { db: QueryCtx["db"] }, after: string | null): Promise` making one indexed read, `ctx.db.query("tenants").withIndex("by_tenant", (q) => (after === null ? q : q.gt("tenantId", after))).first()`, and answering that row's `tenantId`, or `null` when no tenant follows `after`; `null` as `after` asks for the first tenant (E-49, Law 11) - validatorOperator: `const operatorValidator = v.string()` the `operator` argument of every operator entry that changes anything and the type of every field that records it (E-48) -- fnAssertOperator: `assertOperator(operator: string): string` returning its argument, and throwing the plain `Error` `"An operator entry needs a stated operator"` when `operator.trim()` is empty, so a whitespace-only operator of any length is refused as empty, and `"The stated operator is " + bytes + " bytes, above the limit of 512"` when `utf8Length(operator.trim())`, the operator's length after trimming, is above `limitOperatorBytes`; an operator entry calls it first, so a refused entry has read and written nothing (E-48, D19) -- limitOperatorBytes: 512 bytes of UTF-8 after trimming, the bound `limitCallText` of `spec:command.command-pipeline` gives an actor's `id` (E-48, E-34) +- fnAssertOperator: `assertOperator(operator: string): string` returning the trimmed text, which is the operator every caller records, and throwing the plain `Error` `"An operator entry needs a stated operator"` when `operator.trim()` is empty, so a whitespace-only operator of any length is refused as empty, and `"The stated operator is " + bytes + " bytes, above the limit of 512"` when `utf8Length(operator.trim())`, the operator's length after trimming, is above `limitOperatorBytes`; an operator entry calls it first, so a refused entry has read and written nothing (E-48, D19) +- limitOperatorBytes: 512 bytes of UTF-8 after trimming; the trimmed text is the operator, the one checked against the bound and the one recorded, so no stored operator exceeds it; the bound `limitCallText` of `spec:command.command-pipeline` gives an actor's `id` (E-48, E-34) - operatorRecordedWhere: `startedBy` and `changedBy` on the generation row of `spec:application.generation-registry`, `changedBy` on a closed entry of the gate and `operator` on its audit record in `spec:application.write-pause` and `spec:operations.baseline-operations`, and `operator` and `finishedBy` on the run record of `spec:application.restore` (E-48, D19) ## Verification — reviewed diff --git a/src/command/actor-and-scope.ts b/src/command/actor-and-scope.ts index ab320ef..139981f 100644 --- a/src/command/actor-and-scope.ts +++ b/src/command/actor-and-scope.ts @@ -77,9 +77,9 @@ export type AuthorizeInput = { export const operatorValidator = v.string(); // The bound limitCallText gives an actor's id. export const limitOperatorBytes = limitActorIdLength; -// An operator entry calls this first, so a refused entry has read and written nothing. The operator is -// trimmed before both checks, so whitespace alone, of any length, is an empty operator; the argument is -// returned and recorded unchanged. +// An operator entry calls this first, so a refused entry has read and written nothing. The trimmed text +// is the operator: it is what both checks read and what every caller records, so whitespace alone, of +// any length, is an empty operator and no stored operator exceeds the bound. export function assertOperator(operator: string): string { const trimmed = operator.trim(); if (trimmed === "") @@ -89,5 +89,5 @@ export function assertOperator(operator: string): string { throw new Error( `The stated operator is ${bytes} bytes, above the limit of ${limitOperatorBytes}`, ); - return operator; + return trimmed; } diff --git a/src/gate/entries.ts b/src/gate/entries.ts index d1560e7..c838b67 100644 --- a/src/gate/entries.ts +++ b/src/gate/entries.ts @@ -2,7 +2,7 @@ // composition's gate module, so they are reached as internal.gate.closeGate and so on. Each follows the // operator entry contract of spec:command.actor-and-scope: an internal function, so only admin access // reaches it, that reads no grant and no ctx.auth; the two that change the gate check the stated -// operator first and record it. An operator entry closes and resumes only "all" or a tenant scope; a +// operator first and record the trimmed text it returns. An operator entry closes and resumes only "all" or a tenant scope; a // source scope is closed and reopened by the rebuild alone. Every refusal is a plain Error. import { internalMutationGeneric, @@ -52,8 +52,8 @@ export const closeGate = internalMutation({ operator: operatorValidator, }, returns: v.null(), - handler: async (ctx, { scopeKey, reason, operator }) => { - assertOperator(operator); + handler: async (ctx, { scopeKey, reason, operator: stated }) => { + const operator = assertOperator(stated); assertOperatorScope("closeGate", scopeKey); const bytes = utf8Length(reason); if (bytes < 1 || bytes > limitGateReasonBytes) @@ -67,8 +67,8 @@ export const closeGate = internalMutation({ export const resumeGate = internalMutation({ args: { scopeKey: v.string(), operator: operatorValidator }, returns: v.null(), - handler: async (ctx, { scopeKey, operator }) => { - assertOperator(operator); + handler: async (ctx, { scopeKey, operator: stated }) => { + const operator = assertOperator(stated); assertOperatorScope("resumeGate", scopeKey); await resumeScope(ctx, scopeKey, operator); return null; diff --git a/tests/pure/gate.test.ts b/tests/pure/gate.test.ts index 081fdd4..858d921 100644 --- a/tests/pure/gate.test.ts +++ b/tests/pure/gate.test.ts @@ -26,12 +26,12 @@ test.each(["", " ", "\t\n", "\u2003", " ".repeat(512), " ".repeat(513)])( }, ); -// spec:command.actor-and-scope fnAssertOperator, limitOperatorBytes. +// spec:command.actor-and-scope fnAssertOperator, limitOperatorBytes: the trimmed text is the operator. test.each(["a", " a ", "a".repeat(512), "é".repeat(256), "😀".repeat(128)])( - "pure: a bounded operator is returned unchanged, %j", + "pure: a bounded operator is returned trimmed, %j", (operator) => { expect(limitOperatorBytes).toBe(512); - expect(assertOperator(operator)).toBe(operator); + expect(assertOperator(operator)).toBe(operator.trim()); }, ); diff --git a/tests/pure/write-pause.test.ts b/tests/pure/write-pause.test.ts index 00bd948..62b7348 100644 --- a/tests/pure/write-pause.test.ts +++ b/tests/pure/write-pause.test.ts @@ -10,7 +10,7 @@ import { scopesOfUseCase, } from "../../src/gate/index.js"; -test("pure: assertOperator refuses an empty or whitespace-only operator and one above 512 bytes, and returns any other unchanged", () => { +test("pure: assertOperator refuses an empty or whitespace-only operator and one above 512 bytes, and returns any other trimmed", () => { for (const operator of ["", " ", "\t\n "]) expect(() => assertOperator(operator)).toThrow( "An operator entry needs a stated operator", @@ -25,15 +25,15 @@ test("pure: assertOperator refuses an empty or whitespace-only operator and one expect(() => assertOperator("é".repeat(256) + "x")).toThrow( "The stated operator is 513 bytes, above the limit of 512", ); - expect(assertOperator(" ops-1 ")).toBe(" ops-1 "); + expect(assertOperator(" ops-1 ")).toBe("ops-1"); }); -test("pure: assertOperator trims before both checks, so whitespace of any length is empty and the bound counts the trimmed bytes", () => { +test("pure: the trimmed text is the operator, so whitespace of any length is empty and the bound counts the trimmed bytes", () => { expect(() => assertOperator(" ".repeat(600))).toThrow( "An operator entry needs a stated operator", ); const padded = " ".repeat(300) + "x".repeat(512) + "\t"; - expect(assertOperator(padded)).toBe(padded); + expect(assertOperator(padded)).toBe("x".repeat(512)); expect(() => assertOperator(" " + "x".repeat(513) + " ")).toThrow( "The stated operator is 513 bytes, above the limit of 512", ); diff --git a/tests/simulator/gate.test.ts b/tests/simulator/gate.test.ts index a49cb4a..23a5d69 100644 --- a/tests/simulator/gate.test.ts +++ b/tests/simulator/gate.test.ts @@ -140,8 +140,8 @@ test("convex-test: only an explicitly matching generation exempts an entry", asy }); }); -// spec:application.write-pause fnCloseScope, fnResumeScope, validatorClosedEntry; spec:operations.baseline-operations tableOperatorAudit. -test("convex-test: close and resume record exactly one audit each with unchanged operators", async () => { +// spec:application.write-pause fnCloseScope, fnResumeScope, validatorClosedEntry; spec:operations.baseline-operations tableOperatorAudit; spec:command.actor-and-scope limitOperatorBytes, the trimmed text is the operator. +test("convex-test: close and resume record exactly one audit each with the trimmed operator", async () => { const t = gateApp(); const before = Date.now(); expect(await close(t)).toBeNull(); @@ -155,7 +155,7 @@ test("convex-test: close and resume record exactly one audit each with unchanged scopeKey: "tenant:t", reason: "repair", changedAt: expect.any(Number), - changedBy: operator, + changedBy: operator.trim(), }, ], updatedAt: expect.any(Number), @@ -170,7 +170,7 @@ test("convex-test: close and resume record exactly one audit each with unchanged kind: "gate.close", scopeKey: "tenant:t", reason: "repair", - operator, + operator: operator.trim(), recordedAt: row!.updatedAt, }, ]); @@ -188,7 +188,7 @@ test("convex-test: close and resume record exactly one audit each with unchanged kind: "gate.close", scopeKey: "tenant:t", reason: "repair", - operator, + operator: operator.trim(), recordedAt: row!.updatedAt, }, { @@ -393,7 +393,7 @@ test("convex-test: resumeGate copies the removed entry's generation and reason t scopeKey: "tenant:t", reason: "restored entry", generationId, - operator, + operator: operator.trim(), recordedAt: expect.any(Number), }, ]); diff --git a/tests/simulator/write-pause.test.ts b/tests/simulator/write-pause.test.ts index fc66b3f..08047e8 100644 --- a/tests/simulator/write-pause.test.ts +++ b/tests/simulator/write-pause.test.ts @@ -503,6 +503,34 @@ describe("the operator entries", () => { ); }); +describe("the stated operator", () => { + test( + name( + "a padded operator past 512 bytes whose text fits is stored trimmed on the closed entry and on both audit records", + ), + async () => { + const t = app(); + const padded = " ".repeat(300) + "ops-1" + "\t".repeat(300); + await t.mutation(internal.gate.closeGate, { + scopeKey: "all", + reason: "release", + operator: padded, + }); + expect((await gateDocuments(t))[0]?.closed).toMatchObject([ + { scopeKey: "all", changedBy: "ops-1" }, + ]); + await t.mutation(internal.gate.resumeGate, { + scopeKey: "all", + operator: padded, + }); + expect((await operatorAudit(t)).map(({ operator }) => operator)).toEqual([ + "ops-1", + "ops-1", + ]); + }, + ); +}); + describe("a grant under the gate", () => { const grantArgs = (tenantId: string) => ({ tenantId, From a459b4f0e0ab2c20c1f8607fd82a2973862b8d45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:18:37 +0200 Subject: [PATCH 055/155] ReceiveStock takes the stock item ID bound of PlaceOrder One 64-byte bound for every command that names a stock item, refused invalidInput at step 1 with the same message and details. A native example refuses ReceiveStock one byte past the bound; the PlaceOrder refusal example receives only the stock items within the bound. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 13 +-- design/specs/application.pack.sdp.md | 1 + ...ve-stock-item-id-past-bound-refused.sdp.md | 32 ++++++ .../specs/application/first-experiment.sdp.md | 5 +- ...nt.stock-item-id-past-bound-refused.sdp.md | 2 +- .../orders-inventory-example.sdp.md | 4 +- example/convex/ordering.ts | 15 +-- example/convex/receiving.ts | 19 ++++ tests/native/first-experiment-steps.ts | 97 +++++++++++++++++-- ...e-stock-item-id-past-bound-refused.test.ts | 59 +++++++++++ tests/pure/stock-item-id-bound.test.ts | 20 +++- 11 files changed, 236 insertions(+), 31 deletions(-) create mode 100644 design/specs/application/first-experiment.receive-stock-item-id-past-bound-refused.sdp.md create mode 100644 tests/native/first-experiment.receive-stock-item-id-past-bound-refused.test.ts diff --git a/design/README.md b/design/README.md index 81bf5de..5dff5a8 100644 --- a/design/README.md +++ b/design/README.md @@ -2,7 +2,7 @@ ## What this is -The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 225 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. +The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 226 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. The thesis, in the doc's words. Ownership follows bounded contexts. Atomicity follows the business operation. Asynchrony follows a concrete need to defer work. The default operation is one Convex mutation that authorizes a command, makes the domain decision, saves current state with its events, updates essential read models and records the outcome. A second transaction exists only to wait, to spread load, or to reach an external system. The design succeeds when adding domain sophistication does not add infrastructure. @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -225 specs · 5 packs · 123 anchors → 353 nodes · 1194 edges (0 errors, 0 warnings) +226 specs · 5 packs · 124 anchors → 355 nodes · 1198 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -85,7 +85,7 @@ Detail follows the build (decision method, rule 4). Full means interfaces, TypeS | all | `platform`, `laws`, `decisions`, `facts` | [`foundation`](specs/foundation.pack.sdp.md) | 93 | | 0 | `kernel` | [`kernel-and-context`](specs/kernel-and-context.pack.sdp.md) | 7 | | 1 | `context`, `command` | [`kernel-and-context`](specs/kernel-and-context.pack.sdp.md) (19 with the kernel), [`command-pipeline`](specs/command-pipeline.pack.sdp.md) | 12 + 24 | -| 2 | `application`, `operations`, `constraints` | [`application`](specs/application.pack.sdp.md) | 46 | +| 2 | `application`, `operations`, `constraints` | [`application`](specs/application.pack.sdp.md) | 47 | | 3 | `obligations`, `effects` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 22 (of the pack's 43) | | 4 | `processes` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 6 | | 5 | `agents` | [`durable-and-later`](specs/durable-and-later.pack.sdp.md) | 7 | @@ -224,7 +224,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed, with `AuditRecordInput` and the source of each field in the command; the `operatorAudit` table for the gate changes an operator entry makes; Convex's own audit logging recorded as the do-nothing option; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | | E-44 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | `BusinessFailurePolicy` (`carry` or `reject`) per context call, applied in the executor body through the pinned helper `carryOrReject` | | E-45 | [`application.read-models`](specs/application/read-models.sdp.md) | At most 8 component reads per parent query and 4 read-model rows written per command, counted as the rows step 9 inserts, replaces or deletes and checked after each `streams` entry is applied, above which step 9 throws a plain error | -| E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a maximum of 100 lines and a stock item ID of at most 64 bytes on each line, the bounds `PlaceOrder`'s declaration carries; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` and `CancelOrder` call Orders first; the stock item keeps totals only and `CancelOrder` releases the cancelled order's own lines; `orderNotFound`, `orderAlreadyCancelled` and `insufficientAllocation` as `CancelOrder`'s rejections; the order allocation history view that Sc L2-6 builds | +| E-46 | [`application.first-experiment`](specs/application/first-experiment.sdp.md), [`application.orders-inventory-example`](specs/application/orders-inventory-example.sdp.md) | `CancelOrder` as the second lifecycle command, the order summary as the essential read model, a maximum of 100 lines, the bound `PlaceOrder`'s declaration carries, and a stock item ID of at most 64 bytes in every command that names a stock item; `ReceiveStock` as the command that creates stock, the only way a stock item comes to exist; a unit price in whole minor units on each order line; `insufficientStock` as the rejection for short stock; a 16 KiB budget on the stock item stream and the default on the order stream; `PlaceOrder` and `CancelOrder` call Orders first; the stock item keeps totals only and `CancelOrder` releases the cancelled order's own lines; `orderNotFound`, `orderAlreadyCancelled` and `insufficientAllocation` as `CancelOrder`'s rejections; the order allocation history view that Sc L2-6 builds | | E-47 | [`application.first-experiment`](specs/application/first-experiment.sdp.md) | The `Measurement` record, one per measured run, and how each count is captured: commits from the backend's function log, documents and bytes from the `usageStats` of the command's top-level completion record, the calls per context from a pure test of the use case's executor because the log holds no record of a component call inside a mutation, and a count the backend does not expose recorded as a gap; the documents one `PlaceOrder` reads, expected near N + 6 and counted 3N + 5 on the pinned backend, left for the measurement run to explain | | E-48 | [`command.actor-and-scope`](specs/command/actor-and-scope.sdp.md) | The operator entry: an internal function that only admin access reaches, a required `operator` string of 1 to 512 bytes checked by `assertOperator`, no grant read, and the stated operator recorded on the generation row, the gate's entry and audit record, or the restore's run record; a stated operator proves no identity | | E-49 | [`command.actor-and-scope`](specs/command/actor-and-scope.sdp.md) | The tenant list: a `tenants` table written by `insertGrant` with a tenant's first grant and never deleted, read one tenant at a time through `nextTenant` in tenant-ID order by every pass over the whole deployment | @@ -353,14 +353,15 @@ The example domain's second lifecycle command added six examples that verify no | none | `spec:application.orders-inventory-example.cancel-second-context-rejects` | verifies that a release Inventory refuses rolls back the cancel Orders recorded | | none | `spec:application.orders-inventory-example.cancel-without-grant-refused` | verifies that `CancelOrder` requires its own permission | -The bound on an order line's stock item ID added two examples that verify no doc row, each a child of `spec:application.first-experiment` under E-46: +The bound on a stock item ID added three examples that verify no doc row, each a child of `spec:application.first-experiment` under E-46: | Sc | Added example | Reason | |---|---|---| | none | `spec:application.first-experiment.order-at-stock-item-id-bound` | verifies that the maximum order with every stock item ID at its bound is applied and its `OrderPlaced` fits the payload bound | | none | `spec:application.first-experiment.stock-item-id-past-bound-refused` | verifies that a stock item ID one byte past its bound is refused `invalidInput` before any read | +| none | `spec:application.first-experiment.receive-stock-item-id-past-bound-refused` | verifies that `ReceiveStock` refuses a stock item ID one byte past the same bound before any read | -Package D therefore has 28 examples. +Package D therefore has 29 examples. ## Integration notes diff --git a/design/specs/application.pack.sdp.md b/design/specs/application.pack.sdp.md index 884ec62..60f9088 100644 --- a/design/specs/application.pack.sdp.md +++ b/design/specs/application.pack.sdp.md @@ -27,6 +27,7 @@ specs: - spec:application.first-experiment.order-of-max-lines - spec:application.first-experiment.order-at-stock-item-id-bound - spec:application.first-experiment.stock-item-id-past-bound-refused + - spec:application.first-experiment.receive-stock-item-id-past-bound-refused - spec:application.first-experiment.native-acceptance-production-configuration - spec:application.first-experiment.native-contention - spec:application.orders-inventory-example diff --git a/design/specs/application/first-experiment.receive-stock-item-id-past-bound-refused.sdp.md b/design/specs/application/first-experiment.receive-stock-item-id-past-bound-refused.sdp.md new file mode 100644 index 0000000..6f86294 --- /dev/null +++ b/design/specs/application/first-experiment.receive-stock-item-id-past-bound-refused.sdp.md @@ -0,0 +1,32 @@ +--- +id: spec:application.first-experiment.receive-stock-item-id-past-bound-refused +kind: example +altitude: story +readiness: defined +relations: + refines: spec:application.first-experiment + verifies: spec:application.first-experiment +--- +# ReceiveStock refuses a stock item ID one byte past its bound + +E-46 · native tier · production composition · no acceptance row; the example verifies that `ReceiveStock` refuses a stock item ID longer than the bound `PlaceOrder` also carries at step 1, before any read and before the Inventory context is called. + +## Intent + +- outcome: `ReceiveStock` of 100 items whose last stock item ID is 65 bytes of UTF-8 is refused `invalidInput` naming that item, having read and written no document, and nothing is stored. (E-46, D4) + +```gwt +Given the production composition on a native backend +And ReceiveStock is sent {items: 100} stock items whose IDs are {idBytes: 64} bytes of UTF-8, the last one's {lastIdBytes: 65} +And the backend runs with {configuration: "production configuration"} +When {run: "the ReceiveStock command"} runs +Then the caller receives {answer: "the rejection invalidInput"} +And the rejection names line {line: 99} with a stock item ID of {length: 65} bytes against a bound of {limit: 64} +And the command read {readDocuments: 0} documents and wrote {writtenDocuments: 0} +``` + +## Verification — executable + +- Runs in the native tier on the production composition; every test owns its disposable backend. +- After grants and the order summary's first rebuild through to its switch as setup, the test sends one `ReceiveStock` of 100 items with distinct stock item IDs from an ordinary client under a request key, the last ID built from multi-byte UTF-8 characters so that its character count is below its byte count. +- It asserts the `ConvexError` data exactly, `{ kind: "rejection", code: "invalidInput", commandType: "ReceiveStock", message: "Line 99 needs a stock item ID of at most 64 bytes of UTF-8, not 65", details: { line: 99, length: 65, limit: 64 } }`, reads zero documents read and written from the command's own top-level completion record's `usageStats`, and asserts that the receipts and the Inventory context's stream rows and events are unchanged. diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index ad88fa8..2c6a84c 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -115,7 +115,7 @@ The experiment is an application, a test suite and a report. The application is - datasetSizes: 1 line, 10 lines and 100 lines, the maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) - contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with stock for one, so that one is applied and the rest are rejected for short stock or failed by the engine after its bounded retries, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry; the shared stock item is the first line of every order and each other line is on a stock item of that command alone, received once with quantity two, and no caller retries during the cell, while the caller may repeat a command the engine failed under its same request key (First experiment, Sc L1-11, F1, E-5, E-47) - limitOrderLines: 100 lines per `PlaceOrder`, the largest order the example supports and the bound its declaration carries, chosen under the 1 second timeout, the F13 ceilings and the adapter's stream and byte bounds; it is the example's promise and no platform limit, and the experiment reports documents read and written per line (OQ3, F13) -- limitStockItemIdBytes: [extension] 64 bytes of UTF-8 for the stock item ID of each order line, the bound `PlaceOrder`'s declaration carries beside its 100 lines and checks in its refinement, so a longer one is refused `invalidInput` at step 1 of the command pipeline, before any read and before either context is called, as the stock item ID bound of `spec:application.orders-inventory-example` states; it is chosen so that the maximum order's `OrderPlaced` fits the 16,384-byte payload bound of E-12: `getConvexSize` measures the payload at 25 + 100 × (53 + 64) = 11,725 bytes at the bound, where 128 bytes would make 18,125 and the 256 bytes of a stream ID 30,925 (E-46, E-12, OQ3, F13) +- limitStockItemIdBytes: [extension] 64 bytes of UTF-8 for the stock item ID of each order line of `PlaceOrder` and each item of `ReceiveStock`, the bound both declarations carry and check in their refinements, so a longer one is refused `invalidInput` at step 1 of the command pipeline, before any read and before any context is called, as the stock item ID bound of `spec:application.orders-inventory-example` states; it is chosen so that the maximum order's `OrderPlaced` fits the 16,384-byte payload bound of E-12: `getConvexSize` measures the payload at 25 + 100 × (53 + 64) = 11,725 bytes at the bound, where 128 bytes would make 18,125 and the 256 bytes of a stream ID 30,925 (E-46, E-12, OQ3, F13) - limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate read, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer (E-47, F13) - observedMeasurementPins: [extension] the measurement records name the native backend tier, the local backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124, `convex-test` 0.0.60, Node v24.19.0 and Linux 7.2.7-200.fc44.x86_64 on x64; these observations concern that local environment only (E-47, E-15, Sc L2-9) @@ -138,9 +138,10 @@ The experiment is an application, a test suite and a report. The application is Given the production composition on a native backend And an order of {size:"1 line"|"10 lines"|"the maximum"} with stock contention {contention:"absent"|"present"} And the order's stock item IDs are {idBytes:number} bytes of UTF-8, the last line's {lastIdBytes:number} +And ReceiveStock is sent {items:number} stock items whose IDs are {idBytes:number} bytes of UTF-8, the last one's {lastIdBytes:number} And the contention matrix has sizes {sizes:string} and callers {callers:string} And the backend runs with {configuration:"production configuration"|"test configuration"} -When {run:"the PlaceOrder use case"|"the end-to-end path"} runs +When {run:"the PlaceOrder use case"|"the end-to-end path"|"the ReceiveStock command"} runs Then each successful command makes {commits:number} top-level commit And the core path runs {projectionJobs:number} projection jobs And the use case makes {callsPerContext:number} call per context diff --git a/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md b/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md index f3d8ca5..78c30f2 100644 --- a/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md +++ b/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md @@ -29,5 +29,5 @@ And the command read {readDocuments: 0} documents and wrote {writtenDocuments: 0 ## Verification — executable - Runs in the native tier on the production composition; every test owns its disposable backend. -- After grants and the order summary's first rebuild through to its switch as setup, the test receives stock for all 100 stock items of the order through `ReceiveStock`, so the refusal is not short stock, and sends one `PlaceOrder` with one line on each from an ordinary client under a request key. +- After grants and the order summary's first rebuild through to its switch as setup, the test receives stock through `ReceiveStock` for the 99 stock items of the order within the bound, the last one being an ID `ReceiveStock` refuses too, so a refusal later than step 1 would be short stock and not `invalidInput`, and sends one `PlaceOrder` with one line on each from an ordinary client under a request key. - It asserts the `ConvexError` data exactly, `{ kind: "rejection", code: "invalidInput", commandType: "PlaceOrder", message: "Line 99 needs a stock item ID of at most 64 bytes of UTF-8, not 65", details: { line: 99, length: 65, limit: 64 } }`, reads zero documents read and written from the command's own top-level completion record's `usageStats`, which places the refusal before the grant read and before either context's sub-transaction, and asserts that the receipts, both contexts' stream rows and events and the order summaries are unchanged. diff --git a/design/specs/application/orders-inventory-example.sdp.md b/design/specs/application/orders-inventory-example.sdp.md index cabbb6b..471384b 100644 --- a/design/specs/application/orders-inventory-example.sdp.md +++ b/design/specs/application/orders-inventory-example.sdp.md @@ -49,7 +49,7 @@ The second lifecycle command is `CancelOrder`. The stock item keeps two totals a - **cancel operation** — The Orders context's list-shaped operation `cancel({ orders: [{ orderId }] })`, planning one `cancel` stream command per order with no expected version, so the order's own state decides it, and declaring `maxStreams` 32 as the place operation does; it returns `{ orders: [{ orderId, lines }] }`, the lines of each cancelled order taken from its DTO (D10, E-46). - **receive operation** — The Inventory context's list-shaped operation `receive({ items })`, planning one stock item stream per distinct stock item, which creates the stream of a stock item that has none and adds the quantity to the quantity on hand; it is the only way a stock item comes to exist (Law 1, D10, E-46). - **PlaceOrder** — The composed use case that records an order in Orders and allocates every line in Inventory in one mutation, with one receipt and one outcome, calling Orders first because it creates the order, so a UI double submit is answered `entityExists` before Inventory decides, and Inventory second; it rejects when stock is short, and an order with no line, or with a line whose stock item ID is longer than the stock item ID bound, is refused `invalidInput` at step 1 of the command pipeline, before any read and before either context is called (D10, D4, E-39, E-46, Sc L1-4). -- **ReceiveStock** — The command that receives stock, one call to the Inventory context's receive operation with one receipt and one outcome; every stock item of the experiment, a test's and a measurement's included, is created through it (Law 1, D10, E-46). +- **ReceiveStock** — The command that receives stock, one call to the Inventory context's receive operation with one receipt and one outcome; every stock item of the experiment, a test's and a measurement's included, is created through it, and an item whose stock item ID is longer than the stock item ID bound is refused `invalidInput` at step 1 of the command pipeline, before any read and before the Inventory context is called (Law 1, D10, E-46). - **CancelOrder** — The second lifecycle command: the composed use case `cancelOrder`, input `{ orderId: string }`, that calls the Orders context's cancel operation once with the one order and then the Inventory context's release operation once with the order ID and the stock item and quantity of each line Orders returned for that order, in one mutation with one receipt and one outcome, and answers `{ orderId: string; released: { stockItemId: string; quantity: number }[] }`, the release operation's lines; Orders comes first because the order's state refuses a cancel before Inventory decides, and the quantities are read from what Orders returned in this mutation, never from a read model (D10, D4, Law 9, E-39, E-46). - **CancelOrder declaration** — `name: "CancelOrder"`, `contractVersion: 1`, `permission: { permission: "orders.cancel" }`, `writes` the Orders `order` and the Inventory `stockItem`, `readModels` the order summary from the Orders `order`, `rejections: ["orderNotFound", "orderAlreadyCancelled", "insufficientAllocation"]` and no `bounds`, because the input holds no list, the streams the command writes are its one order and the stock items of that order's lines, at most the 100 `PlaceOrder` admitted, and the order ID is bounded at 256 bytes as every stream ID is (D6, D10, E-38, E-46). - **permissions** — `orders.place`, which `PlaceOrder` requires, `orders.cancel`, which `CancelOrder` requires, `inventory.receive`, which `ReceiveStock` requires, and the read permissions `orders.read`, which the parent queries over orders and the order summary require, and `inventory.read` (D11, E-37, E-46). @@ -69,7 +69,7 @@ The second lifecycle command is `CancelOrder`. The stock item keeps two totals a - **order allocation history** — A history-dependent cross-stream read model of the parent keyed by order ID, folded as a `HistoryProjection` from `OrderPlaced` in Orders and `StockAllocated` and `AllocationReleased` across the stock item streams the order touched, holding per line when it was allocated and released; it is bound once per source stream type, rebuilds only under the write pause, and exists in the experiment for Sc L2-6, not as an essential read (E-9, E-46, D9, Sc L2-6). - **journal inspection** — The context queries that return a stream's events and its current version for a reviewer, used by the experiment and the restore checks (First experiment, D2). - **maximum order size** — The largest number of lines `PlaceOrder` accepts, 100, the bound its declaration carries: the example's promise and no platform limit, chosen by the command's author under the adapter's stream and byte ceilings (OQ3, E-46). -- **stock item ID bound** — The longest stock item ID an order line of `PlaceOrder` may carry, 64 bytes of UTF-8, the bound `PlaceOrder`'s declaration carries beside its 100 lines and checks in its refinement: an order whose line has a longer stock item ID is refused `invalidInput` at step 1 of the command pipeline, before any read and before either context is called, with the message "Line I needs a stock item ID of at most 64 bytes of UTF-8, not N" and the details `{ line, length, limit }` of the first such line, `line` counting from 0; the bound exists because `OrderPlaced` carries every line and `append` holds its payload to the 16,384 bytes of the events-stay-small bound: `getConvexSize` measures the payload `{ lines, total }` at 25 bytes plus, for each line, 53 bytes and its stock item ID's UTF-8 bytes, every number counting 9 bytes whatever its value, so 100 lines at 64 bytes make 25 + 100 × (53 + 64) = 11,725 bytes and fit, while 100 lines at 128 bytes would make 18,125 bytes and at the 256 bytes a stream ID may have 30,925, which the Orders context's `append` refuses as a technical failure inside the Orders call; 64 is therefore the largest power of two at which the maximum order fits (OQ3, E-12, E-46). +- **stock item ID bound** — The longest stock item ID the example accepts, 64 bytes of UTF-8, the bound of every command that names a stock item, which its declaration carries and checks in its refinement: `PlaceOrder` on each order line, beside its 100 lines, and `ReceiveStock` on each item it receives, because a stock item whose ID no order may name could never be ordered and has no use; a command whose line or item has a longer stock item ID is refused `invalidInput` at step 1 of the command pipeline, before any read and before any context is called, with the message "Line I needs a stock item ID of at most 64 bytes of UTF-8, not N" and the details `{ line, length, limit }` of the first such entry, `line` being its position from 0 in the order's lines or the received items; the bound exists because `OrderPlaced` carries every line and `append` holds its payload to the 16,384 bytes of the events-stay-small bound: `getConvexSize` measures the payload `{ lines, total }` at 25 bytes plus, for each line, 53 bytes and its stock item ID's UTF-8 bytes, every number counting 9 bytes whatever its value, so 100 lines at 64 bytes make 25 + 100 × (53 + 64) = 11,725 bytes and fit, while 100 lines at 128 bytes would make 18,125 bytes and at the 256 bytes a stream ID may have 30,925, which the Orders context's `append` refuses as a technical failure inside the Orders call; 64 is therefore the largest power of two at which the maximum order fits (OQ3, E-12, E-46). ## Example space diff --git a/example/convex/ordering.ts b/example/convex/ordering.ts index 3896c62..92ff9ba 100644 --- a/example/convex/ordering.ts +++ b/example/convex/ordering.ts @@ -9,19 +9,16 @@ import { v, type Infer } from "convex/values"; import { internalCommand, publicCommand, - utf8Length, type CommandDeclaration, } from "../../src/command/index.js"; import { components } from "./_generated/api.js"; import { orderSummary } from "./orderSummary.js"; import { orderLineValidator } from "./orders/streams.js"; +import { stockItemIdRefusal } from "./receiving.js"; export const placeOrderPermission = "orders.place"; export const cancelOrderPermission = "orders.cancel"; // The largest order PlaceOrder accepts: the example's promise, chosen under the adapter's ceilings. export const maxOrderLines = 100; -// The longest stock item ID an order line may carry, in UTF-8 bytes, so that the OrderPlaced of the -// largest order fits the journal's payload bound. -export const maxStockItemIdBytes = 64; const orderSource = { contextId: "orders", streamType: "order" }; const stockItemSource = { contextId: "inventory", streamType: "stockItem" }; const placeOrderInput = v.object({ @@ -43,15 +40,7 @@ export const placeOrderDeclaration: CommandDeclaration< refine: ({ lines }) => { if (lines.length === 0) return { message: "An order needs at least one line" }; - for (const [line, { stockItemId }] of lines.entries()) { - const length = utf8Length(stockItemId); - if (length > maxStockItemIdBytes) - return { - message: `Line ${line} needs a stock item ID of at most ${maxStockItemIdBytes} bytes of UTF-8, not ${length}`, - details: { line, length, limit: maxStockItemIdBytes }, - }; - } - return null; + return stockItemIdRefusal(lines.map(({ stockItemId }) => stockItemId)); }, output: placeOrderResult, permission: { permission: placeOrderPermission }, diff --git a/example/convex/receiving.ts b/example/convex/receiving.ts index cfcf26f..f6734c6 100644 --- a/example/convex/receiving.ts +++ b/example/convex/receiving.ts @@ -5,10 +5,27 @@ import { v, type Infer } from "convex/values"; import { internalCommand, publicCommand, + utf8Length, type CommandDeclaration, } from "../../src/command/index.js"; import { components } from "./_generated/api.js"; export const receiveStockPermission = "inventory.receive"; +// The longest stock item ID the example accepts, in UTF-8 bytes, checked by every command that names +// a stock item: the OrderPlaced of the largest order then fits the journal's payload bound, and no +// stock item is received that an order could never name. +export const maxStockItemIdBytes = 64; +// The first stock item ID longer than the bound, as step 1's refinement answers it. +export function stockItemIdRefusal(ids: readonly string[]) { + for (const [line, id] of ids.entries()) { + const length = utf8Length(id); + if (length > maxStockItemIdBytes) + return { + message: `Line ${line} needs a stock item ID of at most ${maxStockItemIdBytes} bytes of UTF-8, not ${length}`, + details: { line, length, limit: maxStockItemIdBytes }, + }; + } + return null; +} const items = v.object({ items: v.array(v.object({ stockItemId: v.string(), quantity: v.number() })), }); @@ -17,6 +34,8 @@ export const receiveStockDeclaration: CommandDeclaration = { name: "ReceiveStock", contractVersion: 1, input: items, + refine: ({ items }) => + stockItemIdRefusal(items.map(({ stockItemId }) => stockItemId)), output: items, permission: { permission: receiveStockPermission }, writes: [{ contextId: "inventory", streamType: "stockItem" }], diff --git a/tests/native/first-experiment-steps.ts b/tests/native/first-experiment-steps.ts index 60ad700..1452db8 100644 --- a/tests/native/first-experiment-steps.ts +++ b/tests/native/first-experiment-steps.ts @@ -13,12 +13,14 @@ import { expect } from "vitest"; import { api, internal } from "../../example/convex/_generated/api.js"; import { maxOrderLines, - maxStockItemIdBytes, placeOrderDeclaration, placeOrderPermission, } from "../../example/convex/ordering.js"; import { readOrdersPermission } from "../../example/convex/readModels.js"; -import { receiveStockPermission } from "../../example/convex/receiving.js"; +import { + maxStockItemIdBytes, + receiveStockPermission, +} from "../../example/convex/receiving.js"; import type { CompletionRecord } from "../../harness/admin.js"; import type { Backend } from "../../harness/backend.js"; import { ordinaryClient } from "../../harness/clients.js"; @@ -223,7 +225,10 @@ const scheduledAfter = new WeakMap< >(); export async function placeOrderRuns( world: ExperimentWorld, - run: "the PlaceOrder use case" | "the end-to-end path", + run: + | "the PlaceOrder use case" + | "the end-to-end path" + | "the ReceiveStock command", ) { if (run !== "the PlaceOrder use case") throw new Error(`These examples run the PlaceOrder use case, not ${run}`); @@ -449,11 +454,20 @@ export async function stockItemIdBytesAre( ); const lastId = required(lines.at(-1), "the last line").stockItemId; expect(lastId.length).toBeLessThan(utf8Length(lastId)); - await receive(world, lines); + // Only stock item IDs within the bound can be received: a longer one is refused by ReceiveStock too. + await receive( + world, + lines.filter( + ({ stockItemId }) => utf8Length(stockItemId) <= maxStockItemIdBytes, + ), + ); } export async function placeOrderAtIdBoundRuns( world: StockItemIdWorld, - run: "the PlaceOrder use case" | "the end-to-end path", + run: + | "the PlaceOrder use case" + | "the end-to-end path" + | "the ReceiveStock command", ) { expect(run).toBe("the PlaceOrder use case"); const backend = required(world.backend, "the backend"); @@ -518,11 +532,12 @@ export function stockItemIdRejectionIs( line: number, length: number, limit: number, + commandType: "PlaceOrder" | "ReceiveStock" = "PlaceOrder", ) { expect((world.error as ConvexError).data).toEqual({ kind: "rejection", code: "invalidInput", - commandType: "PlaceOrder", + commandType, message: `Line ${line} needs a stock item ID of at most ${limit} bytes of UTF-8, not ${length}`, details: { line, length, limit }, }); @@ -536,3 +551,73 @@ export function commandDocumentsAre( expect(usage.databaseReadDocuments).toBe(readDocuments); expect(usage.databaseWriteDocuments).toBe(writtenDocuments); } +// ReceiveStock of stock items whose IDs have exact byte lengths, the last built from two-byte +// characters so that its character count is below its byte count. +export function receiveItemsWithIdBytes( + world: StockItemIdWorld, + items: number, + idBytes: number, + lastIdBytes: number, +) { + world.lines = orderLines(items, "sku"); + for (const [i, line] of world.lines.entries()) + line.stockItemId = + i === items - 1 + ? (lastIdBytes % 2 === 1 ? "a" : "") + + "é".repeat(Math.floor(lastIdBytes / 2)) + : line.stockItemId.padEnd(idBytes, "x"); + for (const [i, { stockItemId }] of world.lines.entries()) + expect(utf8Length(stockItemId)).toBe( + i === items - 1 ? lastIdBytes : idBytes, + ); + const lastId = required(world.lines.at(-1), "the last item").stockItemId; + expect(lastId.length).toBeLessThan(utf8Length(lastId)); +} +const receiveStockIdentifier = getFunctionName(api.receiving.receiveStock); +export async function receiveStockRuns( + world: StockItemIdWorld, + run: + | "the PlaceOrder use case" + | "the end-to-end path" + | "the ReceiveStock command", +) { + expect(run).toBe("the ReceiveStock command"); + const backend = required(world.backend, "the backend"); + const client = required(world.client, "the client"); + world.before = await storedOrderDocuments(backend); + const mark = await backend.admin.logMark(); + await client + .mutation(api.receiving.receiveStock, { + tenantId, + requestKey: "k-receive-past-id-bound", + input: { + items: required(world.lines, "the items").map(({ stockItemId }) => ({ + stockItemId, + quantity: 2, + })), + }, + }) + .then( + () => { + throw new Error("ReceiveStock past the stock item ID bound applied"); + }, + (error: unknown) => { + world.error = error; + }, + ); + const records = await backend.admin.completionsSince(mark, (records) => + records.some( + (record) => + record.identifier === receiveStockIdentifier && + record.componentPath === null, + ), + ); + world.own = required( + records.find( + (record) => + record.identifier === receiveStockIdentifier && + record.componentPath === null, + ), + "the command's completion record", + ); +} diff --git a/tests/native/first-experiment.receive-stock-item-id-past-bound-refused.test.ts b/tests/native/first-experiment.receive-stock-item-id-past-bound-refused.test.ts new file mode 100644 index 0000000..42c6578 --- /dev/null +++ b/tests/native/first-experiment.receive-stock-item-id-past-bound-refused.test.ts @@ -0,0 +1,59 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect } from "vitest"; +import { receiveStockItemIdPastBoundRefusedContract as contract } from "../../generated/contracts/application.first-experiment.receive-stock-item-id-past-bound-refused.contract.js"; +import { required } from "../../harness/native.js"; +import { + application, + commandDocumentsAre, + configurationIs, + receiveItemsWithIdBytes, + receiveStockRuns, + stockItemIdAnswerIs, + stockItemIdRejectionIs, + storedOrderDocuments, + type StockItemIdWorld, +} from "./first-experiment-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:application.first-experiment.receive-stock-item-id-past-bound-refused", + ), + verifies: ref( + "spec:application.first-experiment.receive-stock-item-id-past-bound-refused", + ), +}); +void anchor; +bindExample( + contract, + (): StockItemIdWorld => ({}), + { + "the production composition on a native backend": (world) => + application(world), + "ReceiveStock is sent {items} stock items whose IDs are {idBytes} bytes of UTF-8, the last one's {lastIdBytes}": + (world, { items, idBytes, lastIdBytes }) => + receiveItemsWithIdBytes(world, items, idBytes, lastIdBytes), + "the backend runs with {configuration}": (world, { configuration }) => + configurationIs(world, configuration), + "{run} runs": (world, { run }) => receiveStockRuns(world, run), + "the caller receives {answer}": (world, { answer }) => + stockItemIdAnswerIs(world, answer), + "the rejection names line {line} with a stock item ID of {length} bytes against a bound of {limit}": + (world, { line, length, limit }) => + stockItemIdRejectionIs(world, line, length, limit, "ReceiveStock"), + "the command read {readDocuments} documents and wrote {writtenDocuments}": ( + world, + { readDocuments, writtenDocuments }, + ) => commandDocumentsAre(world, readDocuments, writtenDocuments), + }, + async (world) => { + expect( + await storedOrderDocuments(required(world.backend, "the backend")), + ).toEqual( + required(world.before, "the stored documents before the command"), + ); + }, +); diff --git a/tests/pure/stock-item-id-bound.test.ts b/tests/pure/stock-item-id-bound.test.ts index 9bad149..f740cf0 100644 --- a/tests/pure/stock-item-id-bound.test.ts +++ b/tests/pure/stock-item-id-bound.test.ts @@ -2,9 +2,12 @@ import { ConvexError, getConvexSize, type Value } from "convex/values"; import { expect, test } from "vitest"; import { maxOrderLines, - maxStockItemIdBytes, placeOrderDeclaration, } from "../../example/convex/ordering.js"; +import { + maxStockItemIdBytes, + receiveStockDeclaration, +} from "../../example/convex/receiving.js"; import { orderDecider, type OrderLine } from "../../example/domain/index.js"; import { runPipeline, type MutationCtx } from "../../src/command/index.js"; import { limitPayloadBytes } from "../../src/context/index.js"; @@ -150,3 +153,18 @@ test("pure: the pipeline refuses a long ID before any read and checks the line c } expect(touched).toEqual([]); }); + +test("pure: ReceiveStock refuses the first stock item ID past the same bound", () => { + const refineReceive = receiveStockDeclaration.refine; + if (refineReceive === undefined) + throw new Error("ReceiveStock needs its refinement"); + const items = (ids: string[]) => ({ + items: ids.map((stockItemId) => ({ stockItemId, quantity: 1 })), + }); + expect(refineReceive(items(["a".repeat(64), "é".repeat(32)]))).toBeNull(); + expect( + refineReceive( + items(["a".repeat(64), "a" + "é".repeat(32), "b".repeat(70)]), + ), + ).toEqual(refusal(1, 65)); +}); From 73f506b5e8fae115aad7651f8905c96652d2a1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:19:54 +0200 Subject: [PATCH 056/155] design: the harness's rerun count names an execution, not an attempt Co-Authored-By: Claude Opus 5.5 --- design/specs/platform/native-harness.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 9501279..face3dd 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -94,7 +94,7 @@ The harness is plain test support with no Convex function in it. The entries who - observedWriteRate: [extension] the backend holds and then refuses writes past a rate, with `TooManyWrites` and the text "limited to 4 MiB bytes written per 1 second"; one transaction that writes 8.8 MiB commits, so the rate is not a bound on a transaction; this is why a test that writes more than the rate paces its transactions, and why a throughput measured on this backend states the limit (E-15, S13) - observedFunctionLog: [extension] the backend keeps its last 1000 function log entries in memory and loses them at a restart, and it logs one completion record per top-level call and none for a nested or component call inside it; the harness reads the log by polling `/api/stream_function_logs` from a mark built on the machine clock (E-15) - observedOccRetry: [extension] a completion record of a mutation execution refused at commit on an optimistic-concurrency conflict carries `willRetry` true and an `occInfo` that names the table, its component path and a `retryCount` that numbers the refused executions of the request from zero, even when `error` is null; its written documents and bytes are zero, and the rerun in the same request has its own record; which table `occInfo` names varied between runs of one race, the receipts, events or streams table; the harness type of a completion record leaves these fields out and the tests read them from the record (E-15) -- observedOccRerunCount: [extension] the completion record of a mutation attempt that lost an optimistic-concurrency race carries no error, `error` null, under the same request ID as the rerun that follows, so a count of completion records is a count of executions, and the reruns of one request are its completion records minus one (E-15) +- observedOccRerunCount: [extension] the completion record of a mutation execution that lost an optimistic-concurrency race carries no error, `error` null, under the same request ID as the rerun that follows, so a count of completion records is a count of executions, and the reruns of one request are its completion records minus one (E-15) - observedReadCount: [extension] a completion record's `usageStats.databaseReadDocuments` counts the documents the call read, and a call refused before its handler, by argument validation or by Convex's visibility rule, has a record that read none (E-15) - observedUsageStats: [extension] a top-level mutation's completion record carries `usageStats` with `databaseReadDocuments`, `databaseWriteDocuments`, `databaseReadBytes`, `databaseWriteBytes` and `databaseWriteIndexRows`, and they count what a component call inside the mutation read and wrote (E-15) - observedFailedMutationWrites: [extension] a failed top-level mutation's completion record shows no written documents, `usageStats.databaseWriteDocuments` 0, even when a component call inside it wrote before the throw, so the function log cannot show writes that rolled back (E-15) From e6e11124b713558818eda163d6308692e9caf3ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:28:00 +0200 Subject: [PATCH 057/155] build: the measurement counts the gate and audit tables among the rows a run leaves behind Co-Authored-By: Claude Opus 5.5 --- tests/native/place-order-attribution.ts | 3 ++ tests/native/place-order-contention.ts | 6 ++++ tests/native/place-order-measurement.test.ts | 6 ++++ tests/native/place-order-measurement.ts | 32 +++++++++++++------- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/tests/native/place-order-attribution.ts b/tests/native/place-order-attribution.ts index f4a6b3c..51e4eb5 100644 --- a/tests/native/place-order-attribution.ts +++ b/tests/native/place-order-attribution.ts @@ -28,6 +28,9 @@ export const tables = { generations: ["generations", undefined], receipts: ["receipts", undefined], summaries: ["orderSummaries", undefined], + gates: ["maintenanceGates", undefined], + auditRecords: ["auditRecords", undefined], + operatorAudit: ["operatorAudit", undefined], orderStreams: ["streams", "orders"], orderEvents: ["events", "orders"], stockStreams: ["streams", "inventory"], diff --git a/tests/native/place-order-contention.ts b/tests/native/place-order-contention.ts index 920bba7..60d53a0 100644 --- a/tests/native/place-order-contention.ts +++ b/tests/native/place-order-contention.ts @@ -44,6 +44,9 @@ export async function contentionCell(lines: 1 | 10 | 100, callers: 2 | 8 | 32) { orderEvents: 0, stockStreams: stockItems, stockEvents: stockItems, + gates: 0, + auditRecords: 0, + operatorAudit: 0, }); expect(result.value.rowsAfter).toEqual({ grants: 4, @@ -54,6 +57,9 @@ export async function contentionCell(lines: 1 | 10 | 100, callers: 2 | 8 | 32) { orderEvents: 1, stockStreams: stockItems, stockEvents: stockItems + lines, + gates: 0, + auditRecords: 0, + operatorAudit: 0, }); expect( result.value.outcomeCounts.rejected + result.value.engineFailedCommands, diff --git a/tests/native/place-order-measurement.test.ts b/tests/native/place-order-measurement.test.ts index fd45e05..b92bdcc 100644 --- a/tests/native/place-order-measurement.test.ts +++ b/tests/native/place-order-measurement.test.ts @@ -54,6 +54,9 @@ test.each([1, 10, 100] as const)( orderEvents: 1, stockStreams: lines, stockEvents: 2 * lines, + gates: 0, + auditRecords: 0, + operatorAudit: 0, }); expect( result.after.orderEvents.length - result.before.orderEvents.length, @@ -200,6 +203,9 @@ test.each([ orderEvents: 8, stockStreams: 80, stockEvents: 160, + gates: 0, + auditRecords: 0, + operatorAudit: 0, }); for (const record of result.records.filter((record) => !record.willRetry)) expectUsage(record, healthyUsage[10]); diff --git a/tests/native/place-order-measurement.ts b/tests/native/place-order-measurement.ts index a88099c..5ecdb81 100644 --- a/tests/native/place-order-measurement.ts +++ b/tests/native/place-order-measurement.ts @@ -201,27 +201,37 @@ export async function schedules(backend: Backend) { ); return Object.fromEntries(entries); } -// Rows left behind are counted over every table the production composition deploys; markers, the -// gate and audit records are reported as not deployed only while neither schema declares a table. +// Rows left behind are counted over every table the production composition deploys that holds +// neither state, events, receipts nor read-model rows: the registry, the gate and both audit tables. +// Grants and the tenant list are setup. Markers are reported as not deployed while neither schema +// declares a table for them; a table either schema adds fails this check until it is counted. export function leftBehind(before: Stored, after: Stored) { expect(Object.keys(schema.tables).sort()).toEqual([ + "auditRecords", "generations", "grants", + "maintenanceGates", + "operatorAudit", "orderSummaries", "receipts", + "tenants", ]); expect(Object.keys(contextTables).sort()).toEqual(["events", "streams"]); + const counted = ( + name: "generations" | "gates" | "auditRecords" | "operatorAudit", + ) => ({ + before: before[name].length, + after: after[name].length, + created: after[name].filter( + (row) => !before[name].some((prior) => prior._id === row._id), + ).length, + }); return { - generations: { - before: before.generations.length, - after: after.generations.length, - created: after.generations.filter( - (row) => !before.generations.some((prior) => prior._id === row._id), - ).length, - }, + generations: counted("generations"), markers: "not deployed", - gate: "not deployed", - audit: "not deployed", + gate: counted("gates"), + audit: counted("auditRecords"), + operatorAudit: counted("operatorAudit"), }; } // A complete experiment record, distinct from the harness's name/value Measurement envelope. From a5d339d1b6665ad4332621bc47ccf42301e55cf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:34:36 +0200 Subject: [PATCH 058/155] design: the migrations probe's table-scope example names no parent or context table, because the fixture's table numbering is not the fact Co-Authored-By: Claude Opus 5.5 --- ...its-generation-backfill.probe-6-context-enumeration.sdp.md | 2 +- ...ts-generation-backfill.probe-6-context-enumeration.test.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md index 77f6c10..0776371 100644 --- a/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md +++ b/design/specs/facts/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.sdp.md @@ -30,7 +30,7 @@ Then the parent table walk fails with {parentRows: 0} processed context rows and - The bound values first written expected the parent walk of the context's `streams` table to succeed over zero rows; the first run on the pinned releases reported `failed` with `Uncaught Error: Index streams.by_creation_time not found.`, with and without the schema option, and the example is bound to that observed refusal. - The parent's typed data model refuses the context's table name in the compiled tier; the fixture declares it under an expected type error so that the runtime is reached, and the compiler fails if a later release admits the name. - A parent `customRange` without the schema option fails with `Uncaught Error: You must provide your schema to use a custom range.`, as bound before its run. -- A parent mutation that reads a context row by its ID fails with ``Invalid argument `id` for `db.get`: expected to be an Id<"streams">, got Id<"depthRows"> instead.``, and a context migration that patches a parent row by its ID fails with the same kind of error naming a context table, so an ID is read in the namespace of the component that reads it. +- A parent mutation that reads a context row by its ID fails with ``Invalid argument `id` for `db.get`: expected to be an Id<"streams">, got Id<``, followed by the name of a parent table, and a context migration that patches a parent row by its ID fails with the same kind of error naming a context table, so an ID is read in the namespace of the component that reads it. - The migration mounted in the context completes over the five documents and writes the parent's summaries only through a function handle the parent supplied, one parent call per source row. - The parent migration over page records calls `list` once per batch and writes three page records for five documents at a page of two; it carries two cursors, the component's over the page records and the context's in each record. - Hosted quotas are not answered by this local example. diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.test.ts index be738a9..a0ee59c 100644 --- a/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.test.ts +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-enumeration.test.ts @@ -79,7 +79,7 @@ bindExample(contract, (): World => ({}), { ); record("parent reads context id", parentIdError); expect(parentIdError).toContain( - 'Invalid argument `id` for `db.get`: expected to be an Id<"streams">, got Id<"depthRows"> instead.', + 'Invalid argument `id` for `db.get`: expected to be an Id<"streams">, got Id<', ); await backend.admin.run("migrations:streams", {}, { component: "depot" }); const context = await untilStatus( @@ -117,7 +117,7 @@ bindExample(contract, (): World => ({}), { ); record("context writes parent id", failed); expect(failed.error).toContain( - 'Invalid argument `id` for `db.patch`: expected to be an Id<"documentSummaries">, got Id<"migrationSettings"> instead.', + 'Invalid argument `id` for `db.patch`: expected to be an Id<"documentSummaries">, got Id<', ); expect( (await backend.admin.readTable("documentSummaries"))[0]!.title, From 8fb627c87faae76babcff2aff776f7127132f820 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:39:35 +0200 Subject: [PATCH 059/155] build: deployTemporary stops its deploy child when the caller's signal aborts The snapshot fixture's deploy passes the caller's signal again, joined with the backend's own, and a pure test aborts each one during the deploy. Co-Authored-By: Claude Opus 5.5 --- design/specs/platform/native-harness.sdp.md | 2 +- harness/admin.ts | 13 +++++-- harness/snapshot.ts | 2 +- tests/pure/harness-rules.test.ts | 38 +++++++++++++++++++++ 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index face3dd..72ed4bd 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -69,7 +69,7 @@ The harness is plain test support with no Convex function in it. The entries who - adminReads: [extension] the harness reads a table of the app or of a component, and the function log's completion records, with the admin key; a table read returns every document with every Convex value intact or fails, never a sample, and is several queries, not one snapshot; a log read returns every completion record after a point the test marks or fails, and a mark taken before a restart cannot be read after it; a completion record carries at least `identifier`, `componentPath`, `requestId`, `executionTime` in seconds, `cachedResult`, `error` and `logLines` as an array of strings (E-15) - typeFixtureIssuer: [extension] `interface FixtureIssuer { issuer: string; applicationID: string; jwks: string; token(subject: string): Promise }` (E-13) - typeBackend: [extension] `interface Backend { url: string; adminKey: string; issuer: FixtureIssuer; admin: AdminAccess; stop(): Promise; kill(): Promise; restart(): Promise; dispose(): Promise; facts(): BackendFacts }`; `stop` and `kill` keep the storage; `dispose` alone removes it and is final, so a disposed backend cannot be started again; `adminKey` is there for the one example that shows why no caller carries it (E-15) -- typeAdminAccess: [extension] `interface AdminAccess { deploy(): Promise; deployTemporary(directory: string): Promise<{ command: string[]; stdout: string; stderr: string }>; codegen(): Promise; setEnvironment(variables: Record): Promise; environment(): Promise>; run(name: string, args?: Record, options?: { component?: string }): Promise; readTable(table: string, options?: { component?: string; pageSize?: number }): Promise[]>; writeTable(table: string, write: { insert: Record } | { patch: string; fields: Record } | { delete: string }, options?: { component?: string }): Promise; logMark(): Promise; completionsSince(mark: LogMark, until: (records: readonly CompletionRecord[]) => boolean, timeoutMs?: number): Promise; exportSnapshot(path: string): Promise; replaceSnapshot(path: string): Promise }` holds every use of the admin key and takes no identity; the two snapshot members run the Convex CLI's export and import with replacement against this backend, and a test uses them and never the admin key itself; `deployTemporary` deploys a temporary copy, returns its redacted command and clears the plain composition claim (E-15) +- typeAdminAccess: [extension] `interface AdminAccess { deploy(): Promise; deployTemporary(directory: string, signal?: AbortSignal): Promise<{ command: string[]; stdout: string; stderr: string }>; codegen(): Promise; setEnvironment(variables: Record): Promise; environment(): Promise>; run(name: string, args?: Record, options?: { component?: string }): Promise; readTable(table: string, options?: { component?: string; pageSize?: number }): Promise[]>; writeTable(table: string, write: { insert: Record } | { patch: string; fields: Record } | { delete: string }, options?: { component?: string }): Promise; logMark(): Promise; completionsSince(mark: LogMark, until: (records: readonly CompletionRecord[]) => boolean, timeoutMs?: number): Promise; exportSnapshot(path: string): Promise; replaceSnapshot(path: string): Promise }` holds every use of the admin key and takes no identity; the two snapshot members run the Convex CLI's export and import with replacement against this backend, and a test uses them and never the admin key itself; `deployTemporary` deploys a temporary copy, stops its deploy child when the caller's signal or the backend's own aborts, returns its redacted command and clears the plain composition claim (E-15) - fnOrdinaryClient: [extension] `ordinaryClient(url: string, options?: { token?: string; fetch?: typeof fetch }): ConvexHttpClient` (E-13) - fnOrdinarySocketClient: [extension] `ordinarySocketClient(url: string, options?: { token?: string; webSocket?: typeof WebSocket }): ConvexClient` (E-13) - fnCountingFetch: [extension] `countingFetch(options: { loseResponses: boolean }): { fetch: typeof fetch; requests(): number }` delivers each request and, when it loses responses, throws after the backend has answered (E-15) diff --git a/harness/admin.ts b/harness/admin.ts index 4657e3a..5bcceb7 100644 --- a/harness/admin.ts +++ b/harness/admin.ts @@ -56,7 +56,11 @@ export interface CompletionRecord { } export interface AdminAccess { deploy(): Promise; - deployTemporary(directory: string): Promise; + // The child stops when the caller's signal or the backend's own aborts. + deployTemporary( + directory: string, + signal?: AbortSignal, + ): Promise; codegen(): Promise; setEnvironment(variables: Readonly>): Promise; environment(): Promise>; @@ -215,11 +219,14 @@ export function createAdminAccess( await cli("deploy"); state.deployed = target.composition; }, - async deployTemporary(directory) { + async deployTemporary(directory, signal) { + const signals = [signal, target.signal].filter( + (one): one is AbortSignal => one !== undefined, + ); const result = await snapshotCommand( target, [...commandArgs.deploy], - target.signal, + signals.length === 0 ? undefined : AbortSignal.any(signals), directory, ); state.deployed = undefined; diff --git a/harness/snapshot.ts b/harness/snapshot.ts index e01de14..d404e71 100644 --- a/harness/snapshot.ts +++ b/harness/snapshot.ts @@ -59,7 +59,7 @@ export function deploySnapshotFixture( signal: AbortSignal, ): Promise { signal.throwIfAborted(); - return backend.admin.deployTemporary(directory); + return backend.admin.deployTemporary(directory, signal); } export function createSnapshotAccess( diff --git a/tests/pure/harness-rules.test.ts b/tests/pure/harness-rules.test.ts index 6386525..1e48036 100644 --- a/tests/pure/harness-rules.test.ts +++ b/tests/pure/harness-rules.test.ts @@ -40,6 +40,7 @@ import { redactedBuffer } from "../../harness/child.js"; import { invocation } from "../../harness/evidence.js"; import { createAdminAccess } from "../../harness/admin.js"; import { fixtureComposition } from "../../harness/composition.js"; +import { deploySnapshotFixture } from "../../harness/snapshot.js"; import { localWriteRateBytesPerSecond, paceAfterWrite, @@ -281,3 +282,40 @@ test("pure: a rejected snapshot child retains the abort reason and removes its h runner.mockRestore(); } }); + +test("pure: aborting the caller's signal stops the deploy child of a temporary deployment, as the backend's own signal does", async () => { + // A deploy that never finishes by itself, so only an abort ends it. + const runner = vi.spyOn(child, "runChild").mockImplementation( + (_name, _file, _args, options) => + new Promise((_resolve, reject) => { + const signal = options.signal; + if (signal === undefined) + return reject(new Error("the deploy child got no signal")); + signal.addEventListener("abort", () => reject(signal.reason), { + once: true, + }); + }), + ); + try { + for (const [index, owner] of (["caller", "backend"] as const).entries()) { + const caller = new AbortController(); + const backend = new AbortController(); + const reason = new Error(`the ${owner} aborted the deploy`); + let outcome: unknown = "pending"; + void deploySnapshotFixture( + { admin: testAdmin(backend.signal) }, + "unused", + caller.signal, + ).then( + () => (outcome = "deployed"), + (error: unknown) => (outcome = error), + ); + await vi.waitFor(() => expect(runner).toHaveBeenCalledTimes(index + 1)); + expect(runner.mock.calls[index]![2].slice(1, 2)).toEqual(["deploy"]); + (owner === "caller" ? caller : backend).abort(reason); + await vi.waitFor(() => expect(outcome).toBe(reason), { timeout: 2000 }); + } + } finally { + runner.mockRestore(); + } +}); From c85233d7ba58ca6d06b652c35fa2e7fe0591b55f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:40:54 +0200 Subject: [PATCH 060/155] test: CancelOrder over two stock items, in a second tenant, and the Orders cancel operation over two orders Three convex-test cases on the production composition: both stock items' allocations released, the same stock item IDs of another tenant left as they were, and the context operation's answer for every order it cancels. Co-Authored-By: Claude Opus 5.5 --- tests/simulator/example.test.ts | 143 ++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/tests/simulator/example.test.ts b/tests/simulator/example.test.ts index f566352..12d95f0 100644 --- a/tests/simulator/example.test.ts +++ b/tests/simulator/example.test.ts @@ -737,3 +737,146 @@ test( ).toEqual([]); }, ); + +// Stock on sku-1 and sku-2 in one tenant, and order-1 placed over both, by a caller with every grant +// there; the lines name sku-2 first. +async function placedOverTwoItems(t: App, tenantId: string) { + const user = await caller(t, "user-1", everything, tenantId); + await user.mutation(api.receiving.receiveStock, { + tenantId, + input: { + items: [ + { stockItemId: "sku-1", quantity: 5 }, + { stockItemId: "sku-2", quantity: 7 }, + ], + }, + }); + await user.mutation(api.ordering.placeOrder, { + tenantId, + requestKey: "k-place", + input: { + orderId: "order-1", + lines: [line("sku-2", 3, 200), line("sku-1", 2, 400)], + }, + }); + return user; +} +const stockItems = (t: App, tenantId: string) => + Promise.all( + ["sku-1", "sku-2"].map((streamId) => + t.query(components.inventory.queries.stockItem.get, { + tenantId, + streamId, + }), + ), + ); + +test( + name( + "CancelOrder of an order over two stock items releases each one's allocation in the same commit", + ), + async () => { + const t = productionTest(); + await activate(t); + const user = await placedOverTwoItems(t, "t-1"); + expect(await stockItems(t, "t-1")).toMatchObject([ + { onHand: 5, allocated: 2, version: { version: 2 } }, + { onHand: 7, allocated: 3, version: { version: 2 } }, + ]); + const response = await user.mutation( + api.ordering.cancelOrder, + cancelCall("k-1"), + ); + expect(response).toMatchObject({ + kind: "applied", + result: { + orderId: "order-1", + released: [ + { stockItemId: "sku-2", quantity: 3 }, + { stockItemId: "sku-1", quantity: 2 }, + ], + }, + }); + expect(await stockItems(t, "t-1")).toMatchObject([ + { onHand: 5, allocated: 0, version: { version: 3 } }, + { onHand: 7, allocated: 0, version: { version: 3 } }, + ]); + }, +); + +test( + name( + "CancelOrder in t-2 releases t-2's allocation and leaves the stock items of the same IDs in t-1 as they were", + ), + async () => { + const t = productionTest(); + await activate(t); + await placedOverTwoItems(t, "t-1"); + const user = await placedOverTwoItems(t, "t-2"); + const untouched = await stockItems(t, "t-1"); + expect(untouched).toMatchObject([{ allocated: 2 }, { allocated: 3 }]); + const response = await user.mutation(api.ordering.cancelOrder, { + ...cancelCall("k-1"), + tenantId: "t-2", + }); + expect(response.kind).toBe("applied"); + // The order and both stock items it released, every one in t-2. + expect( + response.versions.map( + ({ tenantId, streamId }: { tenantId: string; streamId: string }) => [ + tenantId, + streamId, + ], + ), + ).toEqual([ + ["t-2", "order-1"], + ["t-2", "sku-2"], + ["t-2", "sku-1"], + ]); + expect(await stockItems(t, "t-2")).toMatchObject([ + { allocated: 0, version: { version: 3 } }, + { allocated: 0, version: { version: 3 } }, + ]); + expect(await stockItems(t, "t-1")).toEqual(untouched); + expect( + await user.query(api.orderQueries.getOrder, { + tenantId: "t-1", + orderId: "order-1", + }), + ).toMatchObject({ status: "placed" }); + }, +); + +test( + name( + "the Orders context's cancel operation over two orders answers both, each with its own lines, in the order asked", + ), + async () => { + const t = productionTest(); + const user = await placedOrder(t); + await user.mutation(api.ordering.placeOrder, { + tenantId: "t-1", + requestKey: "k-place-2", + input: { orderId: "order-2", lines: [line("sku-1", 1, 300)] }, + }); + // The context's own operation, which CancelOrder calls with one order, called here with two. + const cancelled = await t.mutation(components.orders.operations.cancel, { + tenantId: "t-1", + actor: operator, + operation: { + operationId: "cancel-two", + causedBy: { kind: "command", commandType: "Setup" }, + }, + input: { orders: [{ orderId: "order-2" }, { orderId: "order-1" }] }, + }); + expect(cancelled.result).toEqual({ + orders: [ + { orderId: "order-2", lines: [line("sku-1", 1, 300)] }, + { + orderId: "order-1", + lines: [line("sku-1", 2, 250), line("sku-1", 1, 999)], + }, + ], + }); + }, +); From a29bfa37823fd70c37a91c80904d2e3adeb19693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:44:51 +0200 Subject: [PATCH 061/155] design: startGeneration refuses an empty tenant list beside a grant, fillTenants fills it, and a resumed pass keeps its batch size The rebuild and registry Specs state the tenant fill as an operator entry with its own progress row and bounded batches, and resumeGeneration takes no batch size, every pass running at the size on the progress row with limitPurgeBatch as the purge's cap. Co-Authored-By: Claude Opus 5.5 --- .../application/generation-registry.sdp.md | 8 +++-- design/specs/application/rebuild.sdp.md | 35 ++++++++++++------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/design/specs/application/generation-registry.sdp.md b/design/specs/application/generation-registry.sdp.md index 0ebd71d..ac58186 100644 --- a/design/specs/application/generation-registry.sdp.md +++ b/design/specs/application/generation-registry.sdp.md @@ -36,19 +36,21 @@ The registry is two parent tables. The `generations` table records every generat - [non-blocking] Extension E-42, as it touches the registry: a restore brings back a generation row and its progress row at their checkpoint and drops the scheduled batch that was driving them, so the row's fence at inventory time is the restore's proof that the generation is still stranded, and `resumeChain` bumping it is the proof that it is not (E-42, F12) - [non-blocking] Extension E-8, the first generation: the doc says how a generation is rebuilt and not how a read model's first generation comes to exist; the option taken here is that a read model is installed through its first rebuild, generation 1 starting as `building` like any other, so a read model added to a deployment that has history serves that history from its first switch and no transition claims coverage it did not build; a command that finds no generation to write fails as a technical failure and a query that finds no active generation fails the same way (E-8, D9, D19) - [non-blocking] Extension E-40, the checkpoint: the cursor, the batch size and the counts live on the progress row and not on the generation row, because a batch that wrote its checkpoint on the row every command reads would re-run every command in flight on the read model once per batch; the progress row is the one checkpoint of every batch whatever drives it (E-40, F1) +- [non-blocking] Extension E-49, as it touches the registry: on a deployment whose grants predate the tenant list the list is empty while grants exist, and a generation started then visits no tenant; the option taken here is that `startGeneration` refuses then, with a message naming `fillTenants`, the operator entry of `spec:application.rebuild` that fills the list from the grants, and refuses while that fill runs (E-49, E-8, Law 11) ## Contract - Every generation of a read model has one row keyed by read model name and generation number, and generation numbers increase by one per read model, starting at 1 (E-8, D9) - A generation is in exactly one of `building`, `verifying`, `verified`, `active`, `retired`, `aborted` or `purged` (E-8, D9) - At most one generation per read model is `active`, and at most one is in `building`, `verifying` or `verified`; these three states are the in-flight states (E-8, D9) -- The allowed transitions are none to building by `startGeneration`, building to verifying by the backfill batch that ends the enumeration, verifying to verified by the verify batch that ends its pass, verified to active by `switchGeneration`, active to retired by the same `switchGeneration`, retired to verifying by `rollbackGeneration`, building, verifying or verified to aborted by `abortGeneration`, and retired or aborted to purged by the purge batch that ends its pass; no other transition exists, and none leads from nothing to active; `interruptGeneration` and `resumeGeneration` change no state, and `resumeGeneration` continues the pass the progress row names, a started purge of a `retired` or `aborted` generation included (E-8, D9) +- The allowed transitions are none to building by `startGeneration`, building to verifying by the backfill batch that ends the enumeration, verifying to verified by the verify batch that ends its pass, verified to active by `switchGeneration`, active to retired by the same `switchGeneration`, retired to verifying by `rollbackGeneration`, building, verifying or verified to aborted by `abortGeneration`, and retired or aborted to purged by the purge batch that ends its pass; no other transition exists, and none leads from nothing to active; `interruptGeneration` and `resumeGeneration` change no state, and `resumeGeneration` continues the pass the progress row names, a started purge of a `retired` or `aborted` generation included, at the batch size the progress row holds, since it takes no batch size (E-8, D9, E-40) - An operator entry of `spec:application.rebuild` makes the transitions named after it, a scheduled batch makes the other three, and nothing else writes a generation row (E-8, E-48, D19) +- [extension] `startGeneration` refuses, and so makes no transition, while the tenant list of `spec:command.actor-and-scope` holds no row and the `grants` table holds one, with a message that names `fillTenants`, and while the tenant fill of `spec:application.rebuild` is in its pass `fill`, because a generation that walked an empty or half-filled list on a deployment whose grants predate the list would be verified and switched without the rows of the tenants it missed (E-49, E-8, Law 11) - [extension] A read model's first generation is generation 1 in `building`, started by `startGeneration` like any other, and the read model has an active generation only after that generation is verified and switched; on a deployment with no subject the build is one backfill batch and one verify batch for each tenant of the tenant list, and one of each when the list is empty (E-8, D9) - A command that maintains a read model writes the active generation and the in-flight generation if one exists; a retired generation is kept unchanged until it is purged and never written, and a rollback reopens it as `verifying` so that live commands write it again while its verify pass repairs it (D9, E-8) - [extension] Each generation is written with the projection whose version its row records, on the live path and by its batches alike, so the rows of one generation are one truth of one projection version (E-9, E-8, D8) - A query reads the active generation and nothing else; a query over a read model that has no active generation throws a plain error that names the read model, a technical failure, and never answers an empty page (D9, E-8, Law 6) -- [extension] The generation row carries the state, the projection version, the fence and who started and last changed it, and nothing a batch writes on its healthy path except the state that the batch ending a pass sets; the progress row carries the cursor, the batch size and the counts, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) +- [extension] The generation row carries the state, the projection version, the fence and who started and last changed it, and nothing a batch writes on its healthy path except the state that the batch ending a pass sets; the progress row carries the cursor, the batch size and the counts, the batch size set by `startGeneration` and never changed after, so every pass of the generation, a resumed one and its purge included, runs at it, a purge batch capped further by `limitPurgeBatch` of `spec:application.rebuild`, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) - [extension] A generation spans every tenant of the deployment and the registry row names none; the tenant is named on every read-model row and marker, on every context call a batch makes, and in the batch cursor, so no read or write of tenant data runs with an absent tenant as a wildcard, and no generation can be switched for one tenant alone (E-8, D11, Law 11) - An aggregate read model keeps one marker per entity per generation with the contribution last recorded and the source versions it was computed from, and every aggregate write reads and writes the marker in the same mutation as the aggregate row; the marker's `sourceVersions` are compared like a row's, so a DTO older than the marker never adjusts the sum (D9, E-8, E-9) - [extension] A generation whose progress row is in the pass `backfill`, `verify` or `purge` has at most one scheduled batch driving it, identified by the row's fence; a restore brings the rows back at their checkpoint and drops that batch with every pending scheduled function, so the generation is stranded until `resumeChain` bumps the fence and reschedules from the saved cursor, which the restore procedure's pending-work inventory and accepted branch drive for every such generation in `building` or `verifying`, and which the operator drives with `resumeGeneration` for a purge the restore stranded, because a purge belongs to a `retired` or `aborted` generation the inventory does not read (E-42, E-8, F12, D19) @@ -75,7 +77,7 @@ Tables live in the parent's `schema.ts`, spread from the library's `readModelTab - tableProjectionMarkers: `projectionMarkers: defineTable({ tenantId: v.string(), readModel: v.string(), generation: v.number(), entityKey: v.string(), aggregateKey: v.string(), contribution: v.number(), sourceVersions: v.array(streamVersionValidator) })` (E-8, D9, Law 11) - indexMarkersByEntity: `.index("by_entity", ["tenantId", "readModel", "generation", "entityKey"])` (E-8) - indexMarkersByEntityUse: point read by `applyAggregate` before every aggregate write (D9, E-8) -- typeRegistryReader: `type RegistryReader = { db: GenericDatabaseReader }` and `type RegistryWriter = { db: GenericDatabaseWriter }`, the part of a ctx the registry's helpers use, where `ReadModelDataModel` is the data model of `readModelTables`, the schema fragment that holds `generations`, `generationProgress` and `projectionMarkers`; every helper that takes them is generic over the composition's data model, because a database reader is not covariant in its data model (E-8) +- typeRegistryReader: `type RegistryReader = { db: GenericDatabaseReader }` and `type RegistryWriter = { db: GenericDatabaseWriter }`, the part of a ctx the registry's helpers use, where `ReadModelDataModel` is the data model of `readModelTables`, the schema fragment that holds `generations`, `generationProgress`, `projectionMarkers` and the tenant fill's `tenantFill` row of `spec:application.rebuild`; every helper that takes them is generic over the composition's data model, because a database reader is not covariant in its data model (E-8) - typeWritableGeneration: `type WritableGeneration = { role: "active" | "building" | "verifying"; generation: number; projectionVersion: number }` where the role is `active` for the state `active`, `building` for the state `building`, and `verifying` for the states `verifying` and `verified` (E-8, E-9, D9) - fnGenerationsToWrite: `generationsToWrite(ctx: RegistryReader, readModel: string): Promise` making at most two indexed reads, the ranges of `by_read_model_state` from `active` to `building` and from `verified` to `verifying`, throwing `"Read model " + readModel + " has " + a + " active generations and " + b + " in building, verifying or verified"` when they hold two active generations or two in-flight ones, and returning one entry per generation a command writes, the active generation first when there is one, then the in-flight one, and an empty list when the read model has neither; the answer is the same for every tenant (E-8, D9) - fnActiveGeneration: `activeGeneration(ctx: RegistryReader, readModel: string): Promise` making one indexed read and answering `undefined` when the read model has no active generation, which a query over the read model then turns into the plain error `"The read model " + readModel + " has no active generation"`; the answer is the same for every tenant, and the query narrows the rows it reads by its own `tenantId` argument (E-8, D9, Law 11) diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 81340c2..451d9e5 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -59,6 +59,7 @@ The registry, the progress row, the per-entity marker and the gate document are - [non-blocking] Extension E-40: the doc binds resumable batching to `@convex-dev/migrations` and gives no batch size; the component iterates a parent table while the sources of a read model live in a context component. The option taken here: a batch is one mutation with one cursor on the progress row and a bound of at most 100 entities, capped by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget; every batch, the backfill and the verify pass over a context's enumeration and the purge over a parent table, is driven by the platform's own self-scheduled internal mutation, and the rebuild uses the component for nothing, because Probe 6 showed that it walks only the tables of the component it is defined in and drives a context's rows only through a parent internal mutation written by hand against a contract its README does not document; the progress row is the one checkpoint (E-40, E-24, E-9, F17, F13, F1) - [non-blocking] Extension E-8: the registry states, the rollback period, what a rollback does, the gate document, the scope a paused rebuild closes and the reach of a generation are design choices the doc does not make; they are pinned in the two contracts under this workflow (E-8, E-9, E-38, D9) - [non-blocking] Extension E-8, what this workflow does not offer: the online path is pinned for a per-entity read model with one source stream type; an `AggregateProjection` and a row built from more than one source stream type have no rebuild here, and each gets one when a view needs it; the paused path of a history view is given below in outline, its cursor across several sources and its verify pass are that view's own rules, and `rollbackGeneration` refuses a generation that needs a write pause, because nothing would hold its sources while live commands fold onto a row that missed the events since its retirement (E-8, E-9, D9, D18) +- [non-blocking] Extension E-49, a deployment whose grants predate the tenant list: the tenant list is written by `insertGrant`, so on such a deployment it holds no row while `grants` holds many, and a pass over an empty list visits no tenant and still ends; the option taken here is that `startGeneration` refuses an empty list beside a grant with a message naming `fillTenants`, and `fillTenants`, an operator entry, walks the grants in bounded batches through a progress row of its own and inserts each missing tenant once; a list that holds a tenant whose first grant came after the table and misses an older one is not detected by that refusal, so the operator runs `fillTenants` on such a deployment before its first `startGeneration` (E-49, E-48, E-40, Law 11) ## Workflow @@ -78,20 +79,24 @@ The registry, the progress row, the per-entity marker and the gate document are - rule: [extension] Every function that starts or changes a generation is an operator entry of `spec:command.actor-and-scope`: an internal function that records the stated operator on the generation row and reads no grant (E-48, D11, D19) - rule: [extension] A generation spans every tenant of the deployment: its batches take one tenant at a time from `nextTenant` of the tenant list, in tenant-ID order, and name that tenant on every context call, so no call runs with an absent tenant as a wildcard, and the one registry row per read model answers every tenant's queries (D11, Law 11, E-8, E-49) - rule: [extension] A tenant whose row enters the tenant list behind a backfill's cursor is not visited by that pass and needs no visit, because every stream it owns was created after the generation was registered and so was written live; the verify pass starts again at the first tenant and visits it (E-49, E-8, D9) +- rule: [extension] `startGeneration` refuses when the tenant list holds no row while the `grants` table holds one, with a message that names `fillTenants`, because a generation started then would visit no tenant, end its passes and be switched with no row of any tenant (E-49, E-8, Law 11) +- rule: [extension] `fillTenants` is an operator entry of `spec:command.actor-and-scope` that fills the tenant list from the `grants` table: its batches walk the grants in tenant-ID order, one distinct tenant per read, at most `limitTenantFillBatch` tenants per batch, through the `tenantFill` progress row of their own, and insert a tenant's row only where none exists, so each tenant is inserted once however often the walk runs; it reads a grant row only as the record that its tenant exists and never to authorize, and it writes nothing but tenant rows and its progress row (E-49, E-48, E-40, D11, Law 11) +- rule: [extension] A tenant fill and a generation in flight exclude each other: `fillTenants` refuses while a read model of the deployment has a generation in `building`, `verifying` or `verified`, and `startGeneration` refuses while the fill's pass is `fill`, so no tenant whose streams predate a generation enters the list behind that generation's cursor (E-49, E-8, D9, F1) - rule: A backfill batch racing a live command is ordered by optimistic concurrency; the batch's read-then-write of a row is one mutation, so the batch never overwrites a row a command wrote after the batch read its source (D9, F1) - rule: [extension] Every batch, whether its source is a context's enumeration or a parent table, is driven by a self-scheduled internal mutation under one batch contract, and the rebuild uses `@convex-dev/migrations` for nothing (E-40, F17, F8) - rule: [extension] A projection version change is a new generation, never an in-place rewrite of the active one (E-8, D8) -- rule: [extension] The batch bound is a field of the progress row, a whole number of at least 1 and at most `rebuildBatchCeiling`, and never more than the context's `list` page admits at the stream type's document budget; only `resumeGeneration` changes it (E-40, E-24, E-9, F1, F13) +- rule: [extension] The batch bound is a field of the progress row, a whole number of at least 1 and at most `rebuildBatchCeiling`, and never more than the context's `list` page admits at the stream type's document budget; `startGeneration` sets it and nothing changes it after, so every pass and every resumed pass runs at the size on the row, and a purge batch deletes at most that many rows, capped by `limitPurgeBatch` (E-40, E-24, E-9, F1, F13) - rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them returns without writing (E-40, E-8, F8) - An operator calls `startGeneration` for a read model; it inserts the generation row in state `building` with the next generation number, the projection version and fence 1, inserts the progress row in the pass `backfill` with the batch size and a `null` cursor, and schedules the first backfill batch, all in one mutation; a read model with no generation gets generation 1 this way (D9, F8, E-8) +- [extension] On a deployment whose grants predate the tenant list an operator calls `fillTenants` before the first `startGeneration`; it inserts the `tenantFill` row, or reopens it, in the pass `fill` with a new fence and schedules the first fill batch; each fill batch reads the gate, takes up to `limitTenantFillBatch` further tenants from the grants after the row's cursor, inserts the tenant row of each that has none, saves the last tenant ID as the cursor and schedules the next fill batch, and the batch that finds no further tenant sets the pass to `idle` and schedules nothing (E-49, E-48, E-40, F8) - From that commit every use case that maintains the read model writes the active generation, when there is one, and the one being built through `applyProjection`, in live-created mode for a stream it created and live-updated mode otherwise (D9, E-9) - Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building`, the pass is not `backfill` or the fence differs, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) -- A batch that throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, and a developer error or a timeout ends the chain; the operator's `interruptGeneration` ends a chain on purpose, and `resumeGeneration` schedules a batch from the saved cursor with a new fence and, when asked, a smaller batch; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) +- A batch that throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, and a developer error or a timeout ends the chain; the operator's `interruptGeneration` ends a chain on purpose, and `resumeGeneration` schedules a batch from the saved cursor with a new fence, at the batch size on the progress row; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) - Each verify batch enumerates the sources again from the progress row's cursor, which the end of the backfill reset to the first tenant, and calls `applyProjection` in backfill mode for every DTO, so a missing row is inserted, an older one updated, and the row of a DTO that projects to `null` deleted; each such repair is counted as a miss; when the last tenant's last page is done the generation moves to `verified` and no batch is scheduled (D9, E-8, Sc L2-5) - `switchGeneration` sets the verified generation `active` and, when the read model has an active generation, sets that one `retired` with `retiredAt` and `retireAfter`, in one mutation; every query reads the active generation from the registry, so every subscription on the read model re-runs (D9, F7, F1) - [extension] During the rollback period the retired generation's rows are kept and not written; `rollbackGeneration` reopens the retired generation it is given as `verifying` with a new fence and a `null` cursor and schedules a verify batch, live commands write it again from that commit, its one verify pass repairs every row that is missing, older or left behind by a deleted subject, and the ordinary `switchGeneration` makes it active and retires the generation that served meanwhile (E-8, D9, Sc L2-5) - `abortGeneration` marks a building, verifying or verified generation `aborted`, leaves the active generation untouched, and resumes every closed gate entry that names this generation; an aborted rollback leaves the reopened generation `aborted`, never `retired` again (D9, Sc L2-6, E-8) -- `purgeGeneration` starts the purge of an aborted generation, or of a retired one once `retireAfter` has passed; each purge batch deletes at most `limitPurgeBatch` rows of the generation for one tenant by the `by_key` range, stays on that tenant while a full batch came back, moves to the next tenant of the tenant list otherwise, and marks the generation `purged` after the last tenant; nothing starts a purge by itself, and a purge whose chain ended, a restore among the causes, is continued by `resumeGeneration` from its progress row (D19, E-40, E-49) +- `purgeGeneration` starts the purge of an aborted generation, or of a retired one once `retireAfter` has passed; each purge batch deletes at most the progress row's batch size, capped by `limitPurgeBatch`, of rows of the generation for one tenant by the `by_key` range, stays on that tenant while a full batch came back, moves to the next tenant of the tenant list otherwise, and marks the generation `purged` after the last tenant; nothing starts a purge by itself, and a purge whose chain ended, a restore among the causes, is continued by `resumeGeneration` from its progress row (D19, E-40, E-49) - For a history-dependent read model the generation row records `pauseRequired`, and `startGeneration` closes the maintenance gate for every source scope of the read model's bindings in the mutation that inserts the row, recording its own generation on each closed entry, so the gate is closed before the first batch is scheduled and this generation's batches are the one background writer allowed through (D9, E-8, E-38) - While the gate is closed for the sources, every use case whose declaration writes one of them and the journal's baseline driver refuse with `writePaused` before any write, whether or not they maintain the view; a batch scheduled before the close reads the closed gate when it starts and refuses, a batch running when the close commits loses the optimistic-concurrency race on the gate document and re-runs into the refusal, and a batch of a superseded chain refuses because its generation fence is stale; nothing needs to drain (D9, D4, E-8, E-38, F1) - Backfill under the pause enumerates a consistent cut because no source changes: each history batch takes one stream from the enumeration, reads one `history` page of it from the saved event cursor, folds the page into the building generation through `applyHistoryProjection` in backfill mode, and saves the stream and the event cursor on the progress row, moving to the next stream when the page reports `isDone`; an interrupted batch resumes from its cursor with the gate still closed, and the gate never reopens by itself (D9, E-9, Sc L2-6) @@ -104,10 +109,10 @@ Each batch is one scheduled internal mutation in the parent. The rebuild schedul The enumeration is the context's `list` query with `includeDeleted`, the one read of context data per batch: one call returns a page of up to `batchSize` DTOs and a continuation cursor, built inside the component by `paginator` from `convex-helpers` over the `by_identity` index that includes deleted subjects, and capped by the query at its byte-derived page bound. A batch reads `page`, `isDone` and `continueCursor` and ignores any split status, because the rows up to `continueCursor` are complete. A DTO carries the stream version the projection compares, as the queries contract's `toDto(state, meta)` maps it onto `version`. Deleted subjects reach the projection as DTOs the projection maps to `null`, so their rows are deleted rather than left behind. - transactionBoundary: each batch is one scheduled internal mutation that commits its rows, its checkpoint and the scheduling of the next batch together; every operator entry that changes anything is one mutation, and `getGenerations` is one query (D9, F8, F9) -- convexSurface: `startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` as the operator entries, `backfillBatch`, `verifyBatch` and `purgeBatch` as the scheduled batches, all internal mutations, and `getGenerations` as an internal query, in the composition's `rebuild` module; the gate entries are in the write-pause contract (D9, E-8, E-48) +- convexSurface: `startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration`, `purgeGeneration` and `fillTenants` as the operator entries, `backfillBatch`, `verifyBatch`, `purgeBatch` and `fillTenantsBatch` as the scheduled batches, all internal mutations, and `getGenerations` as an internal query, in the composition's `rebuild` module; the gate entries are in the write-pause contract (D9, E-8, E-48) - typeRebuildTarget: `type RebuildTarget = { readModel: ReadModel; source: SourceRef; list: FunctionReference<"query", "internal", ListArgs, PaginationResult> }` one per-entity read model the deployment can rebuild, its one source stream type, and the reference to that stream type's `list` query in its context component, such as `components.orders.queries.order.list` (E-8, E-9, D10) -- typeRebuildConfig: `type RebuildConfig = { targets: readonly RebuildTarget[]; refs: { backfillBatch: BatchRef; verifyBatch: BatchRef; purgeBatch: BatchRef } }` where `type BatchRef = FunctionReference<"mutation", "internal", { generationId: Id<"generations">; fence: number }, null>` and `refs` are the composition's own registered batches, which a batch schedules by reference (E-40, F8) -- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; then it inserts the generation row with the next number, `state: "building"`, `pauseRequired: false`, since a `RebuildTarget` is a per-entity read model and the paused start of a history view is given only in outline, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `pass: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48) +- typeRebuildConfig: `type RebuildConfig = { targets: readonly RebuildTarget[]; refs: { backfillBatch: BatchRef; verifyBatch: BatchRef; purgeBatch: BatchRef; fillTenantsBatch: FillBatchRef } }` where `type BatchRef = FunctionReference<"mutation", "internal", { generationId: Id<"generations">; fence: number }, null>`, `type FillBatchRef = FunctionReference<"mutation", "internal", { fence: number }, null>` and `refs` are the composition's own registered batches, which a batch schedules by reference (E-40, E-49, F8) +- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; the tenant list check, throwing `"The tenant list is empty while grants exist; fillTenants fills it from the grants table"` when `nextTenant(ctx, null)` answers `null` and `ctx.db.query("grants").first()` answers a row, and `"The tenant list is being filled; startGeneration waits until fillTenants has ended"` when the `tenantFill` row is in the pass `fill`; then it inserts the generation row with the next number, `state: "building"`, `pauseRequired: false`, since a `RebuildTarget` is a per-entity read model and the paused start of a history view is given only in outline, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `pass: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48, E-49) - fnBatchSizeFor: `batchSizeFor(readModel: ReadModel, requested?: number): number` throwing `"The batch size must be a whole number of at least 1, and " + requested + " is not"` unless `requested` is absent or an integer of at least 1, and returning `min(requested ?? rebuildBatchCeiling(readModel), rebuildBatchCeiling(readModel))`, so a batch size of 0, a negative one and a fraction are refused and a larger one is lowered to the ceiling (E-40, F13) - fnBackfillBatch: `export const backfillBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9, E-40) - fnVerifyBatch: `export const verifyBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9) @@ -120,21 +125,26 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - step5: for each DTO of the page it calls `applyProjection(ctx, target.readModel, { tenantId, dto, versions: [dto.version], mode: "backfill", generations: [thisGeneration] })` for this generation alone, adding one to `rowsWritten` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` for `skipped-newer` or `unchanged` (D9, E-9) - step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) - verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `pass` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and the progress row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) -- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(limitPurgeBatch)`, deletes each row and adds their number to `rowsPurged`; when it deleted `limitPurgeBatch` rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`; a pass that is not done checkpoints and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) +- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(Math.min(batchSize, limitPurgeBatch))` with `batchSize` the progress row's, deletes each row and adds their number to `rowsPurged`; when it deleted that many rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`; a pass that is not done checkpoints and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) - purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) - fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) -- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string, batchSize?: number): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, setting the progress row's `batchSize` to `batchSizeFor(readModel, batchSize)` when one is given, clearing `lastError`, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) -- fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain`, for a generation in any state whose progress row names a pass, so it continues a backfill or verify pass of a generation in `building` or `verifying` and a started purge of a `retired` or `aborted` one, which is how an operator continues a purge a restore stranded (D9, Sc L2-5, E-40, E-48) +- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, clearing `lastError` and leaving the progress row's `batchSize` as it is, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) +- fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain`; it takes no batch size, so a resumed pass runs at the progress row's `batchSize`, a purge capped by `limitPurgeBatch`; for a generation in any state whose progress row names a pass, so it continues a backfill or verify pass of a generation in `building` or `verifying` and a started purge of a `retired` or `aborted` one, which is how an operator continues a purge a restore stranded (D9, Sc L2-5, E-40, E-48) - fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) - fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `pass` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, the in-flight error of `startGeneration` when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0` and no `lastError`, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) - fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and `"Generation " + n + " of " + name + " is " + state + "; abortGeneration needs building, verifying or verified"` unless the state is one of the three; otherwise patching the row to `state: "aborted"`, fence plus 1, `changedAt` and `changedBy`, patching the progress row to `pass: "idle"` and `lastError: "aborted: " + reason`, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8, E-48) - fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged; resumeGeneration continues it"` when `pass` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `pass: "purge"`, `cursor: null` and no `lastError`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) - fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(v.object({ generation: generationDoc, progress: progressDoc })), handler })` the newest `limitGenerationsListed` generations of the read model by `by_read_model` in descending order, each with its progress row, where `generationDoc` and `progressDoc` are the two tables' document validators with `_id` and `_creationTime`; an operator entry that only reads, so it takes no operator (E-8, E-40, E-48, D19) +- tableTenantFill: `tenantFill: defineTable({ pass: v.union(v.literal("fill"), v.literal("idle")), fence: v.number(), cursor: v.union(v.string(), v.null()), batchesDone: v.number(), tenantsRead: v.number(), tenantsInserted: v.number(), lastError: v.optional(v.string()), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), updatedAt: v.number() })` in `readModelTables`, at most one row per deployment, the progress row of the tenant fill and nothing else's; operational data of the deployment with no tenant dimension and no index, read with `ctx.db.query("tenantFill").first()`; `cursor` is the last tenant ID the walk passed, `null` before its first, `tenantsRead` counts the tenants the walk found and `tenantsInserted` the rows it inserted (E-49, E-40, E-48) +- fnFillTenants: `export const fillTenants = internalMutation({ args: { operator: v.string() }, returns: v.null(), handler })` in order: `assertOperator`; for each target of `config.targets` the in-flight check by `by_read_model_state`, throwing `"Read model " + name + " has generation " + n + " in " + state + "; fillTenants needs no generation in flight"`; then, with `now` the mutation's `Date.now()`, when no `tenantFill` row exists it inserts one with `pass: "fill"`, `fence: 1`, `cursor: null`, every count 0, `startedAt`, `changedAt` and `updatedAt` at `now` and `startedBy` and `changedBy` the operator, and when one exists it patches it to `pass: "fill"`, fence plus 1, the cursor and the counts kept when the pass was `fill` and `null` and 0 when it was `idle`, no `lastError`, `changedAt`, `changedBy` and `updatedAt`; then it schedules `refs.fillTenantsBatch` with the row's fence through `ctx.scheduler.runAfter(0, ...)`; so a call continues a fill whose chain a restore, an error or a closed gate ended, and a call after a finished fill walks the grants again and inserts only what is missing (E-49, E-48, E-40, F8) +- fnFillTenantsBatch: `export const fillTenantsBatch = internalMutation({ args: { fence: v.number() }, returns: v.null(), handler })` (E-49, E-40) +- fillSteps: `fillTenantsBatch` loads the `tenantFill` row and returns without writing when it is absent, `pass` is not `fill` or `fence` differs from its argument; it calls `gateAllows(ctx, ["all"])` and, when the answer is not allowed, parks as step 3 does, patching `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt` and scheduling nothing, so the fill waits for `fillTenants` after the gate reopens; otherwise it repeats at most `limitTenantFillBatch` times: it reads `ctx.db.query("grants").withIndex("by_principal", (q) => (cursor === null ? q : q.gt("tenantId", cursor))).first()`, stops when that answers no row, and otherwise reads `tenants` by `by_tenant` for the grant's `tenantId`, inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, adds one to `tenantsRead` and, for an insert, one to `tenantsInserted`, and sets the cursor to that `tenantId`; a batch that took `limitTenantFillBatch` tenants patches the row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt` and schedules `refs.fillTenantsBatch` with the same fence; a batch that found no further grant patches the row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt` and schedules nothing (E-49, E-40, F8, Law 11) +- limitTenantFillBatch: 8 tenants per fill batch; a grant row's bytes are bounded only by the 1 MiB document limit, so a batch reads at most 8 grant rows, 8 MiB, one per tenant through `by_principal`, at most 8 tenant rows of at most `limitIdLength` bytes of tenant ID each, the gate document and the `tenantFill` row, and writes at most 8 tenant rows and the `tenantFill` row, under the 16 MiB read and write ceilings (E-49, E-40, F13) - errorUnknownGeneration: every entry and every helper that takes a `generationId` throws `"No generation " + generationId` when the row is absent, and a batch returns without writing instead (E-8) -- stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its pass is not `idle`, and the operator answers with `resumeGeneration` and a smaller batch, down to 1 (E-40, F1, F9, Probe 6) +- stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its pass is not `idle`; `resumeGeneration` takes no batch size, so the operator answers a stalled backfill or verify pass with `abortGeneration` and a new `startGeneration` with a smaller `batchSize`, down to 1, and a purge batch deletes rows of a generation nothing else writes (E-40, F1, F9, Probe 6) - occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch; a batch reads the generation row and never writes it outside the end of a pass, and it writes the progress row, which no command reads, so a batch's checkpoint re-runs no command (F1, D9, Probe 6, E-40) - limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) -- limitPurgeBatch: `min(500, floor(4194304 / rowBudgetBytes))` rows deleted per purge batch, 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) +- limitPurgeBatch: `min(500, floor(4194304 / rowBudgetBytes))` rows, the cap on the rows a purge batch deletes, which deletes the progress row's `batchSize` when that is smaller; 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) - limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch returns without writing (E-40, F13) - limitBuildingGenerations: at most one generation in `building`, `verifying` or `verified` per read model at a time, so a command writes at most two generations, the active one and that one (E-8, E-9) - limitRollbackPeriod: `[extension]` 7 days by default, stored as `retireAfter` on the retired row; the retired generation is kept and not written, and a rollback at or after `retireAfter` is refused (E-8, D9) @@ -143,7 +153,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - historyStep3: for a history view a batch replaces steps 4 and 5: when the cursor names no stream it calls `list` with `numItems` 1 from the page cursor to take the next stream and sets `eventCursor` to `null`; it then calls `history` for that stream with `limitHistoryBatch` events from `eventCursor`, calls `applyHistoryProjection` in backfill mode for the building generation with the page, and saves `eventCursor`, or clears `streamId` and advances the page cursor when the page reports `isDone` (D9, E-9, E-24, E-40) - historyVerify: a verify batch of a history view enumerates the sources with `list` and, per stream, reads its last event through `history` with `fromVersion` at the stream's version and `numItems` 1, finds that event's row by `keyOf`, and counts a miss when the row is absent or its `sourceVersions` record a lower version for the stream; a miss is repaired by folding the stream's history in place through the same history batches from `eventCursor` null (D9, E-9, Sc L2-6) - rebuildClass: a read model declares its class at declaration time, per entity for the online path, as a `ReadModel` with its projections, and history across streams for the write pause, as a `HistoryProjection`; `pauseRequired` on the generation row records it, and a history view never rebuilds online because a live fold onto a row backfill has not reached cannot be told apart from a complete one (D9, D18, E-9) -- limitHistoryBatch: [extension] `min(500, floor(4 MiB / (2 * rowBudgetBytes)))` events per history batch, 128 at the default 16 KiB row budget and 32 at the 64 KiB ceiling, so one batch reads at most that many events of at most `limitEventBytes` of `spec:context.event-envelope`, 20,480 bytes and the 61 bytes of a stored document's system fields, about 2.5 MiB, one stream row through `list`, and at most one history row per event, at most 2 MiB, and writes at most 2 MiB, under the ceilings with the 1 second timeout in mind; the count is stored on the progress row as its `batchSize` and lowered by `resumeGeneration` like the online bound (E-40, E-9, E-24, F13) +- limitHistoryBatch: [extension] `min(500, floor(4 MiB / (2 * rowBudgetBytes)))` events per history batch, 128 at the default 16 KiB row budget and 32 at the 64 KiB ceiling, so one batch reads at most that many events of at most `limitEventBytes` of `spec:context.event-envelope`, 20,480 bytes and the 61 bytes of a stored document's system fields, about 2.5 MiB, one stream row through `list`, and at most one history row per event, at most 2 MiB, and writes at most 2 MiB, under the ceilings with the 1 second timeout in mind; the count is stored on the progress row as its `batchSize` by `startGeneration` and never changed after, like the online bound (E-40, E-9, E-24, F13) ## Example space @@ -169,4 +179,5 @@ And writes {writes:"never stopped"|"reopen only after verification"|"reopen afte - A reviewer confirms that the gate is read inside every writer's mutation, not cached, that every batch reads it through `gateAllows` and parks its generation instead of throwing, that `startGeneration` closes one entry per source of a paused view and nothing else, and that the write-pause examples assert a refusal of a source writer while closed, the fold of the history view against the journal, and a reopen only after verification. - A reviewer confirms that no registry function, batch or query takes a tenant scope for a generation, that every batch takes its tenant from `nextTenant` and names it on its context call, and that L2-5's example seeds more than one tenant and asserts both are covered at cutover. - A reviewer confirms that `resumeChain` is the only code path that reschedules a stranded generation and that the restore's `finishRestoreRun` calls it for every generation its inventory lists in `building` or `verifying`. +- A reviewer confirms that `fillTenants` and `fillTenantsBatch` write only tenant rows and the `tenantFill` row, read a grant only through `by_principal` to find the next tenant, insert a tenant row only after a point read finds none, and that `startGeneration` refuses an empty tenant list beside a grant and a fill in its pass `fill`, and `fillTenants` a generation in flight. - A reviewer confirms that no function writes a retired generation, that `rollbackGeneration` is the only transition out of `retired` besides purge, and that L2-5's example asserts that the reopened generation is `verified`, holds a row for a subject created since its retirement and none for a subject deleted since, before it is active. From 19250f4af178ddab5ef1c906e902c86fc2e449a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:45:20 +0200 Subject: [PATCH 062/155] design: OD-044's lean rests on the commit timestamp probe, and OD-057's on the migrations probe and the rebuild's own batch driver Both through decisions.py apply; OD-044 stays sorted and the owner's, OD-057 stays waiting under its advisor. Co-Authored-By: Claude Opus 5.5 --- design/decisions/register.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/design/decisions/register.json b/design/decisions/register.json index 4de1d90..1f1d993 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -1761,7 +1761,7 @@ "reason": "Rule 1: the doc says no deployment-wide order token exists [docs/convex-transactional-domain-platform-decisions.md:108] and the commit-timestamp page says one does; the doc against a Convex source resolves nothing without the owner, and D2's reason is the doc's text. Blocks stays L3, where D18's ordered consumer and the history view weigh it.", "lean": { "do": "Keep D2 and the stream-local order; add the commit timestamp to the fact ledger as a documented fact with a probe on the pinned backend before L3 weighs it for D18 and the history view, and let the owner rule whether D2's reason and the doc's line change.", - "restsOn": "The doc: order exists only within a stream, no deployment-wide order token exists, and timestamps, _creationTime and UUID order never prove that a consumer has seen every committed event [docs/convex-transactional-domain-platform-decisions.md:108], carried by D2 and the journal [design/specs/decisions/d02-context-owns-state-and-journal.sdp.md:45; design/specs/context/journal.sdp.md:51]. The Convex page docs.convex.dev/database/advanced/commit-timestamp, read 2026-10-02: 'A commit timestamp represents the time (in nanoseconds) when a mutation commits', 'With an index on db.vars.commitTs ... the timestamp is strictly increasing', 'Mutations execute concurrently before committing, so _creationTime does not necessarily reflect the order in which the mutations committed', and 'it is not possible to schedule a function with the commit timestamp as an argument ... you cannot return the commit timestamp placeholder from a query'. The pinned package, convex 1.46.0, declares db.vars.commitTs as a placeholder that 'resolves at commit to an int64 (bigint) ordered by commit order' and that reading the field back within the writing mutation yields the placeholder [node_modules/convex/dist/esm-types/server/database.d.ts:308-318]. No file under node_modules/convex/dist/esm-types/server/*.d.ts declares getSnapshotTs, which the ledger's finding names, a pointer [design/reviews/consensus-ledger.json:3762]. The doc's rule stays right for _creationTime and recordedAt: recordedAt is Date.now() frozen for the call [design/specs/context/event-envelope.sdp.md:67]. **Judgment.** The engine now documents an order token that costs no counter document and no contended row, which is the cost D2 and Law 11's counter rule were written against; whether a reader can take an upper bound to prove it has seen every committed row is the half the pinned package does not show, so the ledger's claim about a snapshot timestamp is a lead until a page or the package carries it on the pin.", + "restsOn": "The doc: order exists only within a stream, no deployment-wide order token exists, and timestamps, _creationTime and UUID order never prove that a consumer has seen every committed event [docs/convex-transactional-domain-platform-decisions.md:108], carried by D2 and the journal [design/specs/decisions/d02-context-owns-state-and-journal.sdp.md:45; design/specs/context/journal.sdp.md:51]. The Convex page docs.convex.dev/database/advanced/commit-timestamp, read 2026-10-02: 'A commit timestamp represents the time (in nanoseconds) when a mutation commits', 'With an index on db.vars.commitTs ... the timestamp is strictly increasing', 'Mutations execute concurrently before committing, so _creationTime does not necessarily reflect the order in which the mutations committed', and 'it is not possible to schedule a function with the commit timestamp as an argument ... you cannot return the commit timestamp placeholder from a query'. The pinned package, convex 1.46.0, declares db.vars.commitTs as a placeholder that 'resolves at commit to an int64 (bigint) ordered by commit order' and that reading the field back within the writing mutation yields the placeholder [node_modules/convex/dist/esm-types/server/database.d.ts:308-318]. No file under node_modules/convex/dist/esm-types/server/*.d.ts declares getSnapshotTs, which the ledger's finding names, a pointer [design/reviews/consensus-ledger.json:3762]. The doc's rule stays right for _creationTime and recordedAt: recordedAt is Date.now() frozen for the call [design/specs/context/event-envelope.sdp.md:67]. **Judgment.** The engine now documents an order token that costs no counter document and no contended row, which is the cost D2 and Law 11's counter rule were written against; whether a reader can take an upper bound to prove it has seen every committed row is the half the pinned package does not show, so the ledger's claim about a snapshot timestamp is a lead until a page or the package carries it on the pin. Probe 8, at the native backend tier on the pinned release precompiled-2026-09-28-5c7cb5b with convex 1.46.0: one transaction's rows share one commit timestamp in the parent and in two components, and later transactions get larger ones [design/specs/facts/f18-commit-timestamps.sdp.md:13,25]; D2 and the journal still say no order token for the whole deployment exists [design/specs/decisions/d02-context-owns-state-and-journal.sdp.md:45; design/specs/context/journal.sdp.md:51], and the reader's upper bound getSnapshotTs() is internal and absent from the public declarations, so nothing uses it [design/specs/facts/f18-commit-timestamps.sdp.md:13].", "against": "The do-nothing option holds: nothing built orders across streams, the journal forbids it [design/specs/context/journal.sdp.md:139], and no unit before L3 needs a global order. A commit timestamp inside a component's own tables says nothing about the parent's or another context's commits unless one page says the clock is deployment-wide, which the page read does not say in the sentences quoted; and a value that is a placeholder inside the writing mutation cannot be put in an event's envelope and returned in the same call.", "settles": "A fact Spec with the page and the day; a probe on the pinned backend, precompiled-2026-09-28-5c7cb5b with convex 1.46.0: a commitTs field on the events table in a component, ordered across two components and the parent, read back from a query, with what the pinned package offers as the upper bound; then the owner's ruling on D2's text, since the doc wins.", "changes": "The owner's: docs line 108 and spec:decisions.d02 (the reason, not the decision), spec:context.journal:51 and :139, spec:decisions.d18 and the history view when L3 weighs it. For the lean alone: one new fact Spec under design/specs/facts/ and the probe plan. Blocks L3.", @@ -2235,7 +2235,7 @@ "reason": "Rule 3: which Convex mechanism drives a backfill batch, settled by a probe and taken back by an edit [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:22; design/README.md:219].", "lean": { "do": "Run Probe 6 in S4 before the rebuild is built on the component, and until it passes keep E-40's split: @convex-dev/migrations for parent-table batches and a self-scheduled internal mutation for context-enumerated ones.", - "restsOn": "F17 is assumed: the component's page states batches with saved progress and resume, and not batch size or a race with a live command [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:13,29]. Probe 6 must show a backfill batch racing a live command under optimistic concurrency, resume after interruption, and the component driving a batch whose source is a context's enumeration [design/specs/facts/probe-plan.sdp.md:63; design/specs/application/rebuild.sdp.md:57]. E-40 already carries the split until Probe 6 shows the component can [design/README.md:219]. Mutations are serializable under optimistic concurrency, the assumption the race rests on [design/specs/context/persistence-adapter.sdp.md:38]. **Judgment.** The component buys checkpointing and resume that a generation row and a self-scheduled mutation can also keep; a mechanism that restates what a scheduled mutation gives needs the probe to earn its dependency.", + "restsOn": "F17 is assumed: the component's page states batches with saved progress and resume, and not batch size or a race with a live command [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:13,29]. Probe 6 must show a backfill batch racing a live command under optimistic concurrency, resume after interruption, and the component driving a batch whose source is a context's enumeration [design/specs/facts/probe-plan.sdp.md:63; design/specs/application/rebuild.sdp.md:57]. E-40 already carries the split until Probe 6 shows the component can [design/README.md:219]. Mutations are serializable under optimistic concurrency, the assumption the race rests on [design/specs/context/persistence-adapter.sdp.md:38]. **Judgment.** The component buys checkpointing and resume that a generation row and a self-scheduled mutation can also keep; a mechanism that restates what a scheduled mutation gives needs the probe to earn its dependency. Probe 6 ran, at the native backend tier on the pinned release: @convex-dev/migrations 0.3.6 walks only the tables of the component it is defined in, drives a context's rows only through a parent internal mutation written by hand against a contract its README does not document and the compiled tier needs a cast for, keeps its own status apart from the generation's state, and leaves no error on its row when a batch fails past the read limit [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:35,44,49]. So every batch, backfill, verify and purge alike, is driven by the platform's own self-scheduled internal mutation, the rebuild uses the component for nothing, and the progress row is the one checkpoint [design/specs/application/rebuild.sdp.md:58,86,129; design/specs/decisions/d09-rebuild-online-by-default.sdp.md:42]. The lean's condition was met in a way it did not foresee: the split it keeps until the probe passes is gone, because the probe showed the component does not fit.", "against": "The do-nothing option is no component at all: one self-scheduled internal mutation for every batch, no new dependency and one driver instead of two; the leads name two checkpoint owners for migrations-driven batches as a hazard [design/STATE.md:276, a pointer]. If Probe 6 passes, the split costs a second driver for nothing.", "settles": "Probe 6 on the native backend at the pinned release, precompiled-2026-09-28-5c7cb5b with convex 1.46.0, in S4: a batch racing a live command on one read-model row, interruption and resume, and a batch over a context's enumeration, recorded as a run record and on F17.", "changes": "spec:facts.f17-migrations-fits-generation-backfill (status), spec:facts.probe-plan, spec:application.rebuild, spec:decisions.d09-rebuild-online-by-default; S4. The alternative, no component, edits spec:application.rebuild and the E-40 row and removes a dependency.", From 40ea25d353c135cafea816ce63e133a50d53d1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 00:52:23 +0200 Subject: [PATCH 063/155] design: the tenant list, the gate and the bounded operations name the tenant fill actor-and-scope names fillTenants as the other writer and reader of tenants, the write pause lists its batch among the gate's background readers, and the bulk-operations constraint lists it among the bounded operations. Co-Authored-By: Claude Opus 5.5 --- design/specs/application/write-pause.sdp.md | 2 +- design/specs/command/actor-and-scope.sdp.md | 6 +++--- design/specs/constraints/bulk-operations-bounded.sdp.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index 9295b02..165f41d 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -37,7 +37,7 @@ The write pause is the maintenance gate every writer in a scope reads inside its ## Contract -- There is one gate document per deployment, read through `assertWritable`, which throws, by the command pipeline at its step 7 and by the grant helpers `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` before they write, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states; every command that reaches step 7 reads the gate; a duplicate answered from its receipt does not, because its answer writes nothing (D9, D6, E-8, E-25, E-37, E-42) +- There is one gate document per deployment, read through `assertWritable`, which throws, by the command pipeline at its step 7 and by the grant helpers `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` before they write, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches, its tenant fill's `fillTenantsBatch`, and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states; every command that reaches step 7 reads the gate; a duplicate answered from its receipt does not, because its answer writes nothing (D9, D6, E-8, E-25, E-37, E-42) - A scope is `all`, one tenant, or one source stream type of one context; the gate holds the set of closed scopes, each with a reason, the generation it serves if any, and the operator who closed it (E-8, E-48, D9) - A writer in a closed scope refuses before any write with the transient refusal `writePaused`, whose message carries the scope and the reason; nothing is stored, no operation ID is minted, and the same request key succeeds once the scope is open (D9, D4, Law 6) - A use case is in scope when `all` is closed, when its tenant is closed, or when a stream type its declaration's `writes` names is closed (E-8, E-38, D9) diff --git a/design/specs/command/actor-and-scope.sdp.md b/design/specs/command/actor-and-scope.sdp.md index ef4f9ce..a1cd43e 100644 --- a/design/specs/command/actor-and-scope.sdp.md +++ b/design/specs/command/actor-and-scope.sdp.md @@ -49,8 +49,8 @@ This contract pins the shapes the parent establishes and passes into components: - [extension] An operator entry is an `internalMutation` or `internalQuery`, so only admin access reaches it; it reads no grant, calls neither `establishActor` nor `authorize`, and reads no `ctx.auth` (E-48, D11, D19) - [extension] An operator entry that changes anything takes `operator`, a required string that is not only whitespace and is 1 to 512 bytes of UTF-8 after trimming, checks it with `assertOperator` before any read or write, and records the trimmed text `assertOperator` returns on what it changes: the generation row, the gate's closed entry and its audit record, or the restore's run record, so no stored operator exceeds 512 bytes; an operator entry that only reads takes none (E-48, D19) - [extension] A stated operator is what admin access chose to state: no function establishes an actor of kind `operator` from a credential, no permission names an operator duty, and the kind `operator` of `ActorKind` is carried only by an actor a trusted internal caller states (E-48, D11) -- [extension] The tenant list is the `tenants` table: `insertGrant` inserts a tenant's row when it finds none, in the same mutation as the grant, so every tenant that a command or a query was ever authorized in has a row; the library never deletes a row, and revoking every grant of a tenant leaves its row (E-49, D11, Law 11) -- [extension] Grant rows are inserted only through `insertGrant`, so no tenant can hold a grant without a row in the tenant list (E-49, E-37, D11) +- [extension] The tenant list is the `tenants` table: `insertGrant` inserts a tenant's row when it finds none, in the same mutation as the grant, so every tenant that a command or a query was ever authorized in has a row; the one other writer is the tenant fill of `spec:application.rebuild`, whose `fillTenants` inserts the missing row of each tenant the `grants` table names on a deployment whose grants predate the list; the library never deletes a row, and revoking every grant of a tenant leaves its row (E-49, D11, Law 11) +- [extension] Grant rows are inserted only through `insertGrant`, so no tenant whose first grant came after the tenant list can hold a grant without a row in it; a deployment whose grants predate the list fills it through `fillTenants` of `spec:application.rebuild`, and `startGeneration` refuses an empty list beside a grant (E-49, E-37, D11) - [extension] A grant is written and revoked only while the gate allows its tenant: `insertGrant` and `revokeGrant` read the maintenance gate for `all` and the grant's tenant before anything else, so a closed scope or the restore door refuses the composition's grant mutations with the transient `writePaused` and nothing is stored, the tenant row of a first grant included (E-49, E-37, E-8, D11) - [extension] A pass over the whole deployment, a rebuild's batches, a restore's checks and a receipt sweep among them, takes its tenants from `nextTenant`, one tenant per read in ascending tenant-ID order, and names that tenant on every read and write of tenant data; a tenant whose row is inserted behind a pass's cursor is not visited by that pass (E-49, Law 11, D11) @@ -85,7 +85,7 @@ This contract pins the shapes the parent establishes and passes into components: - accessContextOption: when grants need an audit journal, a dedicated access context owns them and `authorize` reads `ctx.runQuery(components.access.grants.forPrincipal, input)` inside the same transaction; the decision type is unchanged (D2, D11) - limitGrantsRead: at most 500 grants per authorization read; the read takes 501 rows and throws a plain `Error`, a technical failure, when 501 come back, because `.take(n)` alone returns at most `n` rows and never shows whether more exist, so a 501st grant is surfaced and never silently truncated (F13, E-37) - tableTenants: `tenants: defineTable({ tenantId: v.string(), createdAt: v.number() }).index("by_tenant", ["tenantId"])` in `commandTables`, the schema fragment that holds `grants`; one row per tenant, at most `limitIdLength` bytes of tenant ID (E-49, D11, Law 11) -- indexTenantsByTenantUse: the point read of `insertGrant` and the range read of `nextTenant`; no command and no query reads the table (E-49) +- indexTenantsByTenantUse: the point reads of `insertGrant` and of the tenant fill's `fillTenantsBatch` in `spec:application.rebuild`, and the range read of `nextTenant`; no command and no query reads the table (E-49) - fnNextTenant: `nextTenant(ctx: { db: QueryCtx["db"] }, after: string | null): Promise` making one indexed read, `ctx.db.query("tenants").withIndex("by_tenant", (q) => (after === null ? q : q.gt("tenantId", after))).first()`, and answering that row's `tenantId`, or `null` when no tenant follows `after`; `null` as `after` asks for the first tenant (E-49, Law 11) - validatorOperator: `const operatorValidator = v.string()` the `operator` argument of every operator entry that changes anything and the type of every field that records it (E-48) - fnAssertOperator: `assertOperator(operator: string): string` returning the trimmed text, which is the operator every caller records, and throwing the plain `Error` `"An operator entry needs a stated operator"` when `operator.trim()` is empty, so a whitespace-only operator of any length is refused as empty, and `"The stated operator is " + bytes + " bytes, above the limit of 512"` when `utf8Length(operator.trim())`, the operator's length after trimming, is above `limitOperatorBytes`; an operator entry calls it first, so a refused entry has read and written nothing (E-48, D19) diff --git a/design/specs/constraints/bulk-operations-bounded.sdp.md b/design/specs/constraints/bulk-operations-bounded.sdp.md index cfa7e5a..0a892f8 100644 --- a/design/specs/constraints/bulk-operations-bounded.sdp.md +++ b/design/specs/constraints/bulk-operations-bounded.sdp.md @@ -14,7 +14,7 @@ relations: Operations bound · Traces: D19, F13. -Every bulk operation, sweeper and batch has a stated bound, tested against the pinned Convex version, and platform limits are ceilings, not batch sizes. In this package the bounded operations are the backfill, verify and purge batches of a rebuild, the check batches of a restore, an import's batches and the operator's audit pages; each states its bound in its Spec. A batch sized at a platform limit is a batch that times out or fails when the limit moves, so every bound sits well below the 32,000 documents scanned, 16,000 written, 16 MiB read and written, 4,096 index ranges and the one second timeout. +Every bulk operation, sweeper and batch has a stated bound, tested against the pinned Convex version, and platform limits are ceilings, not batch sizes. In this package the bounded operations are the backfill, verify and purge batches of a rebuild, the batches of its tenant fill, the check batches of a restore, an import's batches and the operator's audit pages; each states its bound in its Spec. A batch sized at a platform limit is a batch that times out or fails when the limit moves, so every bound sits well below the 32,000 documents scanned, 16,000 written, 16 MiB read and written, 4,096 index ranges and the one second timeout. ## Intent From 8421d6b5d74fee175ffcaa2a6f0ba37c8466e603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:04:10 +0200 Subject: [PATCH 064/155] design: the close of round 2 of the roadmap STATE.md describes round 2: the twelve fork rulings recorded as the owner's, what each of the nine lanes, the integration, the review and the fold-in showed and at which tier, the code inventory after the gate, audit, operator entries and CancelOrder, the next subject (the online rebuild, 3b, to be discussed with the owner before it is prepared), tactical decisions 40 to 45, the owner queue for sitting two and the round's leads. Two marked lines wait for the second review pass and the quiet timings. ROADMAP.md marks 2a to 2e, 3a and 3c done, adds the polish and the fold-in, moves the history view's build in as 5d by ruling F10, narrows the deferral to the aggregate form, and says what round 2 changed and how round 3 now stands. docs/sdp-feedback.md: a model Spec owns an example space. Co-Authored-By: Claude Opus 5.5 --- design/ROADMAP.md | 35 ++++++++--- design/STATE.md | 147 ++++++++++++++++++++++++++++--------------- docs/sdp-feedback.md | 4 ++ 3 files changed, 127 insertions(+), 59 deletions(-) diff --git a/design/ROADMAP.md b/design/ROADMAP.md index 6ce2b5e..05881d8 100644 --- a/design/ROADMAP.md +++ b/design/ROADMAP.md @@ -1,10 +1,10 @@ # Roadmap: the next units -Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. +Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of round 2 on 2026-10-03. ## How to read it -A unit is one session, as `SESSIONS.md` cuts them, on its own branch and through a pull request. Its kind is `decide` when the owner or an advisor rules first, `design` when it writes Spec text from rulings, `build` when it writes code and tests from Spec text already ruled, and `mixed` when the two cannot be separated. Units are grouped in rounds. The lanes of one round touch disjoint files and may run at once, each on its own branch; a round ends when its owner sitting or its blocking unit closes. Sizes are the advisors' judgment: an evening is what S1 took, a night what S2 took. +A unit is one session, as `SESSIONS.md` cuts them, on its own branch. Since round 2 a lane merges into the one feature branch the owner reviews before anything reaches `main`, tactical decision 42 of `STATE.md`. Its kind is `decide` when the owner or an advisor rules first, `design` when it writes Spec text from rulings, `build` when it writes code and tests from Spec text already ruled, and `mixed` when the two cannot be separated. Units are grouped in rounds. The lanes of one round touch disjoint files and may run at once, each on its own branch; a round ends when its owner sitting or its blocking unit closes. Sizes are the advisors' judgment: an evening is what S1 took, a night what S2 took. The slices of `STATE.md` keep their names. S3 is round 1's attribution and round 2's measurement. S4 is the paper of round 2 and the builds of rounds 3 and 4. S5 is round 5. @@ -16,14 +16,16 @@ The slices of `STATE.md` keep their names. S3 is round 1's attribution and round | 1 | b | Attribute the 3N + 5 reads of `PlaceOrder` under controlled changes. Done (#9) | build | convex | an evening | nothing | | 1 | c | Confirm the thirty-three flagged fixes in the Spec text. Done (#6) | design | product | an evening | nothing | | 1 | d | Scout: does the pinned local backend take a snapshot import with replacement. Done, yes (#7) | build | operator, convex | an afternoon | nothing | -| 2 | a | The S3 measurement: six cells, Measurement records, the six targets, Sc L2-9's concurrency half | build | convex | a night | 1b, OD-046, OD-064 | -| 2 | b | S4 on paper: the per-entity generation lifecycle, the gate and the audit record, the operator entry contract | design | domain, operator | an evening | sitting one | -| 2 | c | `CancelOrder` and `release` in the example domain | mixed | domain | an evening | the owner's answer on allocations | -| 2 | d | Probe 6, before the rebuild is built on the component | build | convex | an evening to a night | OD-057's checked lean applied | -| 2 | e | Probe 7, as far as the local backend can take it, with the scheduled-argument boundary | mixed | operator, convex | an evening | nothing: the import runs locally and admin access reads `_scheduled_functions` | -| 3 | a | The gate, the restore door, and a gate change as audit that fails closed | mixed | operator | an evening | 2b | +| 2 | a | The S3 measurement: six cells, Measurement records, the six targets, Sc L2-9's concurrency half. Done: in every contention cell one command applied and the rest were rejected, the six cost targets held, the timings wait for a quiet run | build | convex | a night | 1b, OD-046, OD-064 | +| 2 | b | S4 on paper: the per-entity generation lifecycle, the gate and the audit record, the operator entry contract. Done, by one designer: installation through the first rebuild, the tenant list, the operator entry contract, S4's open findings 31 to 19 | design | domain, operator | an evening | sitting one | +| 2 | c | `CancelOrder` and `release` in the example domain. Done: totals only, six native examples, a cancel writes six documents | mixed | domain | an evening | the owner's answer on allocations | +| 2 | d | Probe 6, before the rebuild is built on the component. Done: the component does not drive a context's rows as documented, so the rebuild drives every batch itself | build | convex | an evening to a night | OD-057's checked lean applied | +| 2 | e | Probe 7, as far as the local backend can take it, with the scheduled-argument boundary. Done: a fresh import keeps no schedule and an import in place keeps every one; 16 MiB of scheduled arguments per mutation | mixed | operator, convex | an evening | nothing: the import runs locally and admin access reads `_scheduled_functions` | +| 2 | f | Also done in round 2: the stock item ID bound, so the largest order fits the payload bound `append` holds an event to; 64 bytes, a refusal at step 1 | build | domain | under an evening | 2c | +| 2 | g | Also done in round 2: the review's fold-in, the caller's signal in the harness, three `CancelOrder` tests, `fillTenants` and the resumed batch size on paper | mixed | operator, domain | under an evening | the review | +| 3 | a | The gate, the restore door, and a gate change as audit that fails closed. Done in round 2: 106 proof tests written apart from the fix, all passing at the merge, the fix wrong nowhere | mixed | operator | an evening | 2b | | 3 | b | The online rebuild of the order summary, Sc L2-5 at the native tier | build | domain | a night | 2b, 2d, 3a | -| 3 | c | The two harness reads and the fact rows no Spec carries | mixed | convex | a session plus an evening | nothing | +| 3 | c | The two harness reads and the fact rows no Spec carries. Done in round 2: F18 to F20 probed, the "no job" assertions read `_scheduled_functions`; the deployed-function read stays deferred | mixed | convex | a session plus an evening | nothing | | 3 | d | Sitting two: the language, the pass sentence, and the S5 rows | decide | product, domain, operator | an evening of forks, one check, one owner sitting | nothing | | 4 | a | A command's audit fails closed and its diagnostic never aborts, Sc ALL-1 | mixed | operator | an evening | 3a | | 4 | c | The history view's fold, its memory and its rollback, on paper | design | domain | an evening | 2c | @@ -32,6 +34,7 @@ The slices of `STATE.md` keep their names. S3 is round 1's attribution and round | 5 | a | Baselines, migration and the fold bound, Sc L2-7 at the native tier | mixed | domain | a night or more | OD-041 from sitting two, 3b | | 5 | b | The restore procedure and the drill, Sc L2-8 | mixed | operator | unknown until 4b | 4b, 5a | | 5 | c | The first adopter-facing cut: README, the example walkthrough, the package shape | mixed | product | a night | OD-047, OD-048, Sc L2-5 to L2-8 bound | +| 5 | d | The build of the order allocation history view, Sc L2-6 at the native tier, by ruling F10 | build | domain | a night | 3b, 4c | ## Round 1: what blocks nothing @@ -49,6 +52,8 @@ No record is copied into `evidence/` in round 1 or later: tactical decision 35 e ## Round 2: after sitting one +Done on 2026-10-02 and 2026-10-03, as nine lanes merged into `feature/round-2`, then an integration, one review and its fold-in. `STATE.md`, "Where the work stands", says what each showed. The lanes below are kept as they were cut; what differed: the main thread ruled the forks of sitting one on the owner's word before the round, so every lane opened at once; 2b ran as one Fable designer with both subjects, not two, because the gate, the rebuild and the operator entries share surfaces; Probe 6 answered that `@convex-dev/migrations` drives a context's rows only through a hand-written parent mutation against an undocumented contract, so the rebuild drives every batch with its own self-scheduled mutation and uses the component for nothing, and OD-057 waits for its lens; 3a, the gate, was pulled into the round and built from 2b's text with the proof written apart, and 3c, the fact rows, ran beside the other lanes because it needed nothing; a polish lane (2f) and the review's fold-in (2g) came after the merge. + **2a. The measurement.** The contention run at 2, 8 and 32 commands on one stock item, the `Measurement` record per run with the retry count read from the log fields the native harness Spec lists as observed, the comparison with the six targets, the local write rate stated beside any throughput, and the contention run bound to Sc L2-9 as OD-007's lean says. OD-063 and OD-064 get their local numbers. **2b. S4 on paper.** One design unit, two Fable designers with disjoint subjects, Opus writers, a closer. The domain subject: the projection chosen by the version the generation row records (OD-014) and installation through a first rebuild (OD-010), both checked leans, which close the version check correction C1 left; the progress row (OD-071); the verify pass deleting a row whose DTO projects to null; the rulings of OD-068 and OD-069 applied. The operator subject: `AuditRecordInput` and the record an operator action writes, the gate's shapes, and the one operator entry contract written from the rulings on OD-148 and OD-025, so that every later operator function is written against one establishment rule. The tenant list's owner is ruled here too. @@ -59,6 +64,16 @@ No record is copied into `evidence/` in round 1 or later: tactical decision 35 e **2e. Probe 7.** Moved up from round 4 by ruling 2 once the import scout answered yes. One unit for both halves: the five states and the retention of `_scheduled_functions` confirmed on the pinned release, a scheduled argument grown until the backend refuses it, and, where 1d allows, a snapshot of the parent and the two context components imported into a fresh backend with what is there recorded. F12 and F16 move to probed or stay documented with the reason recorded. OD-081, OD-120 and OD-114 get the measurement their `settles` names; OD-115 waits for the Workpool and Workflow half, which comes before L3. The restore Spec's sentence on dropped schedules is already a ledger finding for S5, from the scout. +## Round 3, as it now stands + +The owner discusses the next session's work before it is prepared. In this order: + +1. **3b, the online rebuild of the order summary**, Sc L2-5 at the native tier. It is the next unit. What it needs is in `design-s4-paper.md` in the round 2 folder: package 3b of section 3 and the rulings of section 7. 3a is merged, so `assertOperator`, `nextTenant` and `gateAllows` exist. The files: `src/read-model/projection.ts` (`projections`, `projectionOf`, the three roles of a written generation, the backfill mode), `src/read-model/tables.ts` (the slimmer `generations` and `generationProgress`, with its field `pass`), `src/read-model/generations.ts` (the roles, and `activateFirstGeneration` removed), a new `src/read-model/rebuild.ts` (one helper per entry, `RebuildConfig`, `batchSizeFor`, `resumeChain`), the order summary's `projections`, the composition's registered rebuild functions and the fixture's read models the same. Every setup that ran the first activation installs through `startGeneration`, its batches and `switchGeneration`, and the first-activation example is renamed `first-rebuild-installs-read-model` with its test in one commit. From section 7: `fillTenants` and its batches with the refusal of `startGeneration` on an empty tenant list beside a grant, `resumeGeneration` with no batch size, the backfill mode's rule for a row that projects to `null`, and the message that sends a started purge to `resumeGeneration`. The proof obligations are at the pure, `convex-test` and native tiers, the native one the L2-5 example: 350 orders in two tenants, batch size 1, live commands in both tenants, an interrupt, a resume, a switch and a rollback. It does not build the paused path, the aggregate rebuild, `@convex-dev/migrations` in any form, an automatic stall check, a purge that starts by itself or the restore's `finishRestoreRun`. +2. **3d, sitting two.** The language rows OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added (snapshot, tenant fill, the measurement's words); OD-008 with OD-006; OD-138; OD-041, OD-042 and OD-114, the last with Probe 7's facts; OD-044 with F18 against D2; OD-063 with its local numbers; OD-048, the licence; and the two speed targets of F2 once their timings are taken on a quiet machine. +3. **4a, command audit and the diagnostic.** 3a has built `writeAudit` and step 10, so what is left is the Spec text on which outcomes emit a record and where a duration comes from, the sink, and Sc ALL-1 bound. +4. **4c, the history view on paper.** 2c is done, so `OrderCancelled` and `AllocationReleased` exist for its fold. +5. **4d, the receipts sweep.** The tenant list and `nextTenant` are built. + ## Round 3 **3a. The gate and the door.** The gates table, `gateAllows`, `assertWritable` at step 7, close, resume and get for the `all` and tenant scopes, the read of `MAINTENANCE_MODE` before the gate document, and the audit table with `writeAudit` inside every close, resume and abort. At `convex-test`: a command in a closed tenant is refused and nothing is stored, and a close whose audit insert fails leaves the gate open. On the native backend: the environment switch refuses every write, and two writers race a close. @@ -91,7 +106,7 @@ No record is copied into `evidence/` in round 1 or later: tactical decision 35 e - Layers 3 to 6: at their trigger, after Probe 7, as the doc and tactical decision 3 say. OD-050 to OD-054 and OD-056 stay waiting until an activation record names a consumer. - Probe 3's quota half: once on a hosted deployment after 2a, OD-036. -- The build of the history view (Sc L2-6) and the aggregate form: at a view that needs them, OD-075. 4c gives them their rules first. +- The aggregate form: at a view that needs it, OD-075. The history view is no longer deferred: ruling F10 keeps it in the first experiment, 4c writes its rules and 5d builds it. - Receipt tombstones, the derived mapping and `streamParts`, the import command: when a command needs them, OD-100, OD-113, OD-072. - OD-092's `transactionLimits` probe: before any budget or `maxStreams` is raised; nothing in rounds 1 to 5 raises one. - OD-035, the doc against the probed facts: after 2a, so one doc edit carries every number. diff --git a/design/STATE.md b/design/STATE.md index 97af677..29d2302 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -1,29 +1,52 @@ # State of the design corpus -Written on 2026-10-02 at the close of round 1 of the roadmap, which followed the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. +Written on 2026-10-03 at the close of round 2 of the roadmap, which followed round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. ## Measured at close ``` -202 specs · 5 packs · 101 anchors → 308 nodes · 1097 edges (0 errors, 0 warnings) +232 specs · 5 packs · 130 anchors → 367 nodes · 1222 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings; readiness divergence: [] -open questions: 68 Specs, 144 questions, 10 blocking; extensions registered: 51 -stated readiness: {'defined': 192, 'scoped': 10} -ledger: 275 findings, {'fixed': 144, 'open': 112, 'owner': 1, 'partially-fixed': 18} -open findings by slice: {'L3': 31, 'P': 30, 'S3': 1, 'S4': 31, 'S5': 19} -corpus digest: 6623d62b13b9a088 +open questions: 69 Specs, 147 questions, 9 blocking; extensions registered: 53 +stated readiness: {'defined': 222, 'idea': 1, 'scoped': 9} +ledger: 278 findings, {'fixed': 148, 'open': 103, 'owner': 1, 'partially-fixed': 26} +open findings by slice: {'L3': 31, 'P': 30, 'S3': 1, 'S4': 19, 'S5': 22} +corpus digest: e1a6ac63b9bc1390 protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` +**Round 2 is on `feature/round-2`, not on `main`.** `main` is at `592f1dc`, the close of round 1. The lanes of round 2 branched from `main` or from `feature/round-2` and were merged into `feature/round-2`, which is pushed; CI passed on its pushes at `1b92bd8` and `a5d339d`. No pull request to `main` is opened until the owner has reviewed the branch: tactical decision 42 has the owner's words. The next round's lanes branch from `feature/round-2` and merge back into it. + The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Each unit works on its own branch, named for the unit, so that `main` stays green. The owner gave the session a mandate on 2026-10-01 to merge at this early stage, and said the same day that a pull request per unit was not a good idea while the design is iterated quickly. So from slice S1 on there is no pull request: CI runs on every push, and the session fast-forwards `main` once CI is green on the last commit. Slice S0 was the one unit merged through a pull request. That held until the merge of 2026-10-02. **Slice S2 and the platform review unit are on `main`.** The owner said on 2026-10-02, after the close of the platform review unit: "Please merge to main and push. This is the last change we will do without formal PRs since things are starting to shape up." The session fast-forwarded `main` to `fix/platform-review`, which holds `slice/s2`, and pushed it. The owner's earlier bar for S2 was "high-confidence changes only", with a revert to `9221da1` kept cheap until the owner had seen the result. -From this merge on, a unit reaches `main` through a pull request: tactical decision 32. The review as it was received is in commit `70c15bc`, with paths of the owner's machine in its links. The files were deleted from the tree at the close, and the owner said the same day that the deleted review may stay in the history. +From this merge on, a unit reached `main` through a pull request, tactical decision 32, as round 1's four lanes did. Round 2 reaches it as one branch the owner reviews first, decision 42. The review as it was received is in commit `70c15bc`, with paths of the owner's machine in its links. The files were deleted from the tree at the close, and the owner said the same day that the deleted review may stay in the history. ## Where the work stands -Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, and round 1 of that roadmap is on `main`. +Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, round 1 of that roadmap is on `main`, and round 2 is on `feature/round-2`. + +Round 2 of the roadmap, from the evening of 2026-10-02 to past midnight, on the ThinkPad, as nine lanes merged into `feature/round-2`, then an integration, a review and a fold-in. The owner's words at the opening: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." During the round: "we will need a feature branch"; "i agree with the least noisy and cleanest path forward."; "good note on the load. i have moved this to machine that is not capable enough. but i will relocate to M3 mac tomorrow. hopefully we will manage today."; and on 2026-10-03, "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue", "We can discuss details before preparation of work for the next session." and "We will continue development of this project here on the thinkpad. It is not ideal, but sufficient. M3 Mac is continuting with macos native development." Decisions 40 to 45 hold what followed from them. + +The main thread ruled the forks of sitting one on the owner's word, as F1 to F12 of `rulings-1-round-2.md` in the unit folder: twelve rows decided (OD-046, OD-064 and OD-123 by one answer, OD-153, OD-068, OD-069, OD-148, OD-025, OD-150, OD-151, OD-152, OD-047) and OD-048, the licence, left the owner's. The register tool refused a session ruling on an `owner` row, the owner chose the cleanest path, and the twelve are recorded as decided by the owner, each ruling ending with both of the owner's sentences; the tool is unchanged (decision 40). OD-007 was decided by its advisor, its checked lean as the ruling. What each lane showed, at the tier named: + +- Sitting one. An Opus lead wrote the twelve answers into the rows and the forks and applied F1 to F3 at ten Specs: 100 lines is the largest order `PlaceOrder` supports, the bound its declaration carries and no platform limit; the speed targets of F2 are set for the pinned local backend before the benchmark; Sc L2-9 is read as the local observation, its concurrency half the contention run. A sol check found the twelve ruling texts and the thirteen fork answers exact and six Spec sentences that still contradicted a ruling, all corrected. Open questions fell from 144 to 138 at that lane. +- The S3 measurement (2a). `gpt-6-astra` at high, 20 minutes, one turn, on the native backend, local: ten cost cells and nine contention cells, each writing a `Measurement` record. The healthy order of 1, 10 and 100 lines read and wrote 8/6, 35/24 and 305/204 documents; a duplicate or a changed-input conflict read 5 and wrote none. In every one of the nine contention cells one command applied, N minus 1 were rejected for short stock and the engine failed none: 3, 15 and 63 executions at N of 2, 8 and 32, each loser refused once at commit and rejected on its rerun; the worst cell, 100 lines at 32, read 9,915 documents and 4,195,493 bytes for one applied order. Eight commands on unrelated stock, in one tenant or eight, applied with no rerun. The six cost targets held, the fourth as C1 counts it. A stock state that admin access set ahead of its events let an order for 3 apply against 2 received: a ledger finding for S5 (C2), no check on the command path. The journal tail read and the four-document grant read stay (C3, C4), with their costs on OD-066 and the new OD-154. The timings were taken only under load, and two readings 6 times apart at one line say the load and not the cost, so no speed target is compared yet. +- The S4 paper (2b). One Fable designer, 18.5 minutes, both subjects, Spec text only: a read model installed through its first rebuild, the projection chosen by the version its generation row records, progress on a `generationProgress` row only batches read, one verify pass that inserts, updates and deletes, a switch of one write, rollback as F4 rules it, `interruptGeneration`, the tenant list as the `tenants` table (E-49), the operator entry contract of F6 (E-48), the gate's scopes and refusal, and `AuditRecordInput` pinned. `design-s4-paper.md` in the unit folder has the decisions and the build cut. A sol check at medium in 8 minutes listed fifteen first findings: an Opus lead applied 11, sent 2 to the main thread and left 2 with a reason, found 3 sol did not, and cut sol's ledger draft of 37 items to 25. S4's open findings went from 31 to 19. The main thread's six rulings after the check are section 7 of the note. +- `CancelOrder` and `release` (2c). An Opus builder, then a sol read at medium with 35 mutations. Pure, `convex-test` and six native examples on the production composition: the order's `cancel` and `OrderCancelled`, the stock item's `release` and `AllocationReleased` with totals only (F9), Orders called first, the summary row replaced with `cancelled`, three new rejections. A cancel writes six documents on the native backend, and a refused release writes none. Nothing under `src/` changed. Sol's read led to twelve corrections, each new assertion shown to fail with its rule removed, and found that an `OrderPlaced` of 100 lines with 256-byte stock item IDs is 30,925 bytes, above the 16,384 bytes `append` holds a payload to. +- Probe 6 (2d). Astra at high, two turns, 40 minutes, six native examples on the fixture composition: `@convex-dev/migrations` 0.3.6 walks only the tables of the component it is defined in; it drives a context's rows only through a parent internal mutation written by hand, against a contract its README does not document, which the compiled tier needs a cast for; its status and the generation's state are two vocabularies; a batch that reads past 16 MiB fails as a whole and leaves no error on the component's row. So the rebuild drives every batch itself and uses the component for nothing (G2), F17 is probed, and the component is pinned as a dev dependency for the fixture's probe only. +- Probe 7 (2e). Astra at high, two turns, 54 minutes, five native examples with 60 bound values. Into a fresh backend `convex import --replace` restores every document with its `_id` and `_creationTime` and leaves every `_scheduled_functions` table empty, and a stored scheduler ID still validates while `system.get` returns null. In place it replaces later changes with the exported documents and keeps every scheduler row, including rows made after the export, and a kept pending mutation runs against the restored data. Scheduled arguments are bounded at 16,777,216 bytes per mutation in Convex value bytes; a single call of 8 MiB is accepted, with a warning from 80 percent of 4 MiB. A scan of `_scheduled_functions` fails at 4,092 rows on the 4,096-read limit. F12 stays documented and F16 rechecked, each with the reason; the kept schedules are a ledger finding for S5. Admin access gained `deployTemporary` and the completion's log lines, and a ZIP reader in Node replaced the `unzip` binary. +- The fact rows (3c, pulled into the round). Astra at high, 22 minutes, on the native backend under load. F18 (Probe 8): one commit timestamp per transaction in the parent and in components, larger ones later. F19 (Probes 9 and 11): 1,000 empty nested calls committed in 3.8 to 9.4 s and refused requests ended at 15.2 to 15.7 s on the system-operation budget, while a loop of computation was refused at about 1 s; 2,000 component calls committed and 4,000 were refused. F20 (Probe 10): a page of 32 rows at the 256 KiB budget read 8,390,560 bytes against the 8,388,608-byte bound, because the check runs after the row that reaches it. The "no job" assertions of Sc L2-2 and L2-3 now also read `_scheduled_functions` of the parent and both contexts, with a positive control. +- The gate (3a, pulled into the round). The Opus lane lead wrote the fix and astra at high wrote the proof apart from it from the same Spec text, in 16 minutes: 106 tests (5 compiled, 40 pure, 58 `convex-test`, 3 native), all red before the fix and all passing at the merge. The five disagreements at the merge were each a wrong test, and the fix was wrong nowhere. Built: the `tenants` table, `assertOperator`, `nextTenant`, `src/gate` and `src/audit`, step 7 before the operation is minted, step 10's audit record, and `closeGate`, `resumeGate`, `getGate` and `getGateAudit`. On the native backend the restore door's `MAINTENANCE_MODE` refuses the next command in every tenant with no deploy, a new example, and a close that races commands leaves no command committed that read the gate open: the losers rerun, 16 reruns in 8 trials. By ruling, the grant helpers read the gate too, and the trimmed operator is what is recorded. +- Polish. An Opus lead and sol at medium, 5 minutes. A native reproduction showed E2's premise: a 100-line order with 256-byte stock item IDs passed step 1, `append` threw a plain error inside Orders, and the caller got a bare technical failure with nothing stored. `PlaceOrder` and `ReceiveStock` now refuse a stock item ID above 64 bytes of UTF-8 at step 1 as `invalidInput`: 100 lines at 64 bytes make a payload of 11,725 bytes, at 128 bytes 18,125. On the native backend an order at the bound applied with 305 documents read and 204 written, and one byte past it was refused with none read. Five mutations ran red. The stale README counts were recomputed. +- The integration. An Opus integrator merged eight lane branches into `feature/round-2` on top of sitting one. The conflicts were counts, a few bullets two lanes had both written, and generated files, which one codegen after each merge left unchanged. One test disagreement between two lanes (the measurement's tables left behind against the gate's new tables) was a wrong test. At `e6e1112`, every project in one run with other lanes on the machine: 160 files, 761 tests, 754 passed, 1 failed and 6 skipped; the failure was a literal parent table name in a Probe 6 example that the gate's new table shifted, since fixed. `npm run acceptance` on that record: 31 scenarios passed, none failed or absent, 5 missing (Sc L2-5, L2-6 twice, L2-7, L2-8, which have no verifier yet). +- The review. Astra at xhigh, 13 minutes, on a frozen copy of `1b92bd8`: two findings on an ordinary path (the Probe 6 fixture callback writes a wrong status, `deploySnapshotFixture` drops the caller's signal), four surviving mutations, and two places where the S4 text did not determine the behavior (a deployment whose grants predate the tenant list, the batch size of a resumed purge). With 8 mutations and the gate and polish lanes outside its copy it was thin for xhigh, so a second pass reads the head after the fold-in. +- The second pass of the review, astra at xhigh on a frozen copy of `40ea25d`. `[review pass 2]` +- The fold-in. An Opus agent with no GPT job, because the rulings named every change: the caller's signal reaches the deploy child (pure test), three `CancelOrder` tests at `convex-test` over two stock items, a second tenant and two orders, each shown to fail under its mutation, and the Spec text of `fillTenants`, the refusal of `startGeneration` on an empty tenant list beside a grant, and a resumed pass keeping its batch size. At `40ea25d`: typecheck, lint and format pass, `npm test` 59 files and 615 tests, the six native files that reach `deployTemporary` pass, `check.py` OK. +- The timings on a quiet machine. `[timings]` + +The branches. Each lane worked in `~/dev-libar/wt/` on a branch of its own, from `main` at `592f1dc` for the first seven and from `feature/round-2` or `unit/s4-paper` for gate and polish, and the integrator merged each into `feature/round-2` in `~/dev-libar/wt/round-2`. The branch is pushed. The owner's words: "we will need a feature branch", then "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue". So no pull request goes to `main` until the owner has reviewed `feature/round-2`. Round 1 of the roadmap, on 2026-10-02, in one evening on the second machine, as four lanes on their own branches, merged through pull requests #6 to #9. No Fable agent ran: the owner said the same day that the weekly Fable limit is better spent on implementation, and decisions 36 to 38 hold the rulings. What the round showed, each at the tier named: @@ -96,18 +119,21 @@ What passed, and at which tier, with backend release `precompiled-2026-09-28-5c7 Every Layer 1 scenario runs on the fixture composition. The Layer 2 scenarios run on the production composition, except the one that needs a fault. -What the code has and what it does not have yet, after S2. The Specs pin all of it and say nothing about which part is built: the owner's direction of 2026-10-01 is that a Spec carries no temporal information. +What the code has and what it does not have yet, after round 2. The Specs pin all of it and say nothing about which part is built: the owner's direction of 2026-10-01 is that a Spec carries no temporal information. -- Pipeline: both entries, steps 1 to 6 with the ID length bound, the mint of the `OperationRef` at step 7, step 8, step 9 with the `writes` check, the row budget and the row count, the receipt insert of step 10 and the return of step 11. Not yet: the gate read of step 7, the audit record, the diagnostic. -- Declaration: `writes` is required and `readModels` is optional. No `audit` or `irreversible` field yet, and `retention.afterExpiry` is `"delete"` only. There is no `commandRegistry` and no registration check: the Specs dropped them, and the compiler over the generated api is the check of Sc L2-4. -- Read models: the per-entity form only, written from the DTOs of a command's stream entries. A binding admits a `ReadModel` and nothing else. The first activation is the one generation transition. Not yet: `applyAggregate`, `applyHistoryProjection`, the backfill mode, `projectionMarkers`, `getGenerations`, start, rebuild, switch, rollback and purge. There is no check of a generation's `projectionVersion` against the declared projection's (correction C1). +- Pipeline: both entries, steps 1 to 6 with the ID length bound, the gate read of step 7 through `assertWritable` before the `OperationRef` is minted, step 8, step 9 with the `writes` check, the row budget and the row count, the receipt insert of step 10 with the audit record of a declaration that sets `audit`, and the return of step 11. Not yet: the diagnostic. +- Declaration: `writes` is required, `readModels` and `audit` are optional. No `irreversible` field yet, and `retention.afterExpiry` is `"delete"` only. There is no `commandRegistry` and no registration check: the Specs dropped them, and the compiler over the generated api is the check of Sc L2-4. +- The gate and the audit records: `src/gate` and `src/audit`. The `maintenanceGates`, `operatorAudit` and `auditRecords` tables, `gateAllows` reading `MAINTENANCE_MODE` before the gate document, `assertWritable`, `scopesOfUseCase`, `closeScope` and `resumeScope` with their audit record in the same mutation, `writeAudit` and `writeOperatorAudit` with no catch, and the four operator entries `closeGate`, `resumeGate`, `getGate` and `getGateAudit` that both compositions re-export. Not yet: a source scope closed by `startGeneration`, the restore's entries, the audit queries by tenant, a typed `env`. +- Authority: `establishActor`, `authorize`, the grants table, the `tenants` table, and `insertGrant`, which inserts a tenant's row with its first grant, `revokeGrant` and `nextTenant`; both grant helpers read the gate first. `assertOperator` checks the stated operator of an operator entry and returns its trimmed text. The library registers no grant function; each composition registers its own two internal grant mutations. Not yet: `fillTenants` and its batches, which the rebuild Spec now pins. +- Read models: the per-entity form only, written from the DTOs of a command's stream entries, with one `projection` on the declaration. The code still installs a read model by the first activation, while the Specs now install it through its first rebuild; the first-rebuild example is bound to its steps and skipped. Not yet: the rebuild (`projections` by version, `generationProgress`, `startGeneration`, the backfill and verify batches, switch, interrupt, resume, rollback, abort and purge), `applyAggregate`, `applyHistoryProjection`, `projectionMarkers`, `getGenerations`. - Receipts: the key, the fingerprint, lookup, classification and insert. An expired receipt is treated as absent and deleted in the same mutation. Not yet: tombstones and the sweep. - Outcome boundary: the wire shapes, `reject`, `refuseTransient`, `normalizeThrown` with the closed code list, `classifyThrown`. Not yet: the conditional dispatcher and `carryOrReject`, which has no caller. - Context: operations, `get` and `list` with `includeDeleted` for a trusted caller. Not yet: `history`, `rebuild`, `byOperation` and `maintenance.writeBaseline`. - Queries: a parent query authorizes through `authorizeQuery`, and its refusal is the rejection wire shape with the query's name in `commandType`. A client pages a list by a cursor pair, and no client hook crosses the component boundary. - Journal and adapter: the single mapping only. Not yet: baselines and migrations (step 3a, `migrateToSchemaVersion`, `defineMaintenance`, the driver), `rebuildStream`, the derived mapping and `streamParts`. A stream row behind the registration's `stateSchemaVersion` is a plain error for now. -- Authority: `establishActor`, `authorize`, the grants table and helpers that insert and delete a grant. The library registers no grant function; each composition registers its own two internal grant mutations. -- The example domain: `CancelOrder`, `release`, `OrderCancelled` and `AllocationReleased` are in the Specs and not in the code. `getOrderWithStock` and the parent queries over Inventory are not built, because no Spec pins their shapes; `inventory.read` is seeded and nothing asks for it yet. `installedLayers` of the production composition is `[]`, because the Spec is silent. +- The example domain: `PlaceOrder`, `ReceiveStock` and `CancelOrder`, the order's `cancel` and the stock item's `release`, `OrderCancelled` and `AllocationReleased`, with a stock item ID bound of 64 bytes on `PlaceOrder`'s lines and `ReceiveStock`'s items. `getOrderWithStock` and the parent queries over Inventory are not built, because no Spec pins their shapes; `inventory.read` is seeded and nothing asks for it yet. `installedLayers` of the production composition is `[]`, because the Spec is silent. +- Harness: admin access reads `_scheduled_functions`, exports and imports a snapshot, deploys a temporary copy of a composition with `deployTemporary` and hands back a completion's log lines; a ZIP reader in Node reads the export. +- Not yet, beyond the rebuild: the history view (a unit of round 5 by F10, its rules first in 4c), baselines and migration, the restore procedure, the receipts sweep and the diagnostic. What changed in the Specs during S1, and why: @@ -131,39 +157,39 @@ What changed in the Specs during S2, and why: Review of S2. One review read the whole build at `fd8ad02`, in three lanes that shared nothing: astra at xhigh on behavior, sol at medium on the library, the fixture composition and the harness, sol at medium on the production composition, its tests and the later Spec lines. None found a defect on a path a healthy run or an ordinary failure takes. Astra ran sixteen mutations under `src/`: two survived, and each now has a test. One Spec sentence was wrong against the native backend (two concurrent `PlaceOrder` commands both apply when stock covers both) and was corrected. Two fold-ins followed, `9b20134` with `f3d4a77` for the orchestrator's rulings on the build and `705fb32` for the review. One read of the fold-ins followed, sol at medium at `705fb32`. It found one defect on a healthy path, in a path the first fold-in had added and in the fixture composition: the depot relay that reads one document authorized without the document as its subject, so a reader granted that one document was refused. It also found two tests that would not catch a removal and one stale Spec sentence. All four were fixed in `e76e587`, each new test shown to fail under its removal, and the slice stopped on tactical decision 19 with no further read. The rulings and the reports are in the S2 folder the project notes name. -No `fixed` finding carries `unreviewed: true` any more: round 1 confirmed the thirty-three in the Spec text or carried their remainder into a new open finding. The eight `partially-fixed` findings that carry the flag wait for the slices that build their rest. +Round 1 confirmed the thirty-three flagged `fixed` findings in the Spec text or carried their remainder into a new open finding. Round 2's S4 paper and the main thread's rulings after its check set four findings `fixed` with `unreviewed: true`; they and the seven `partially-fixed` findings that carry the flag are confirmed when the build of their subject touches them. Review round 3 stands as it was for the rest of the corpus: no lens approved. The design is finished by building it, one slice at a time. | Slice | Builds | Open findings | Blocker, major, minor | |---|---|---|---| | S3 | Measurement: 1, 10 and 100 lines, without and with contention. Repeated after S4 and S5 | 1 | 0, 0, 1 | -| S4 | Generations and rebuild, the history view and the write pause, with the tenant list, operator entry points and gate audit the rebuild uses; Probe 6 | 31 | 0, 15, 16 | -| S5 | Baseline migration, restore, the rest of operators and audit, diagnostics, the receipts sweep, maintenance jobs; the transactional part of Probe 7 | 19 | 0, 9, 10 | +| S4 | Generations and rebuild, the history view and the write pause, with the tenant list, operator entry points and gate audit the rebuild uses. The paper, the gate and Probe 6 are done | 19 | 0, 10, 9 | +| S5 | Baseline migration, restore, the rest of operators and audit, diagnostics, the receipts sweep, maintenance jobs. Probe 7's local part is done | 22 | 0, 12, 10 | | L3 | Layers 3 to 6, written from what the experiment shows | 31 | 0, 12, 19 | | P | Polish with no effect on what is built: citations, markers, stale sentences, README | 30 | 0, 0, 30 | -Eighteen more findings are `partially-fixed`, each waiting for its slice. Remove an unsupported generic promise before designing a helper no view needs. +Twenty-six more findings are `partially-fixed`, each waiting for its slice. Remove an unsupported generic promise before designing a helper no view needs. ## Next unit -`ROADMAP.md` is the order, ruled as cut by decision 36, and round 1 is done. Next is sitting one: the owner reads the thirteen forks under `design/decisions/forks/` and answers each in a sentence, and an `owner` unit writes the words into the rows and the forks and applies each ruling at its Spec. Then round 2 opens as `ROADMAP.md` now cuts it: the S3 measurement (2a), the S4 design on paper (2b, the one unit of the round that spends Fable, two designers), `CancelOrder` and `release` (2c), Probe 6 (2d) and Probe 7 (2e, moved up from round 4 because the import runs locally). Only 2c and 2d and 2e can open before the sitting; 2c waits for OD-151's answer unless the owner says otherwise. +`ROADMAP.md` is the order, ruled as cut by decision 36, and rounds 1 and 2 are done. The next subject is the online rebuild of the order summary, roadmap 3b, with Sc L2-5 at the native tier: package 3b of `design-s4-paper.md` in the unit folder names its files, its order, its proof obligations and what it must not build, and section 7 adds the rulings after the check, `fillTenants` and the resumed batch size among them. The owner said on 2026-10-03: "We can discuss details before preparation of work for the next session." So the main thread lays the next session's work out with the owner before any brief is written or any lane starts. -`slice` S3 is round 1's attribution and round 2's measurement, and S3's measurement needs `OD-046` and `OD-064` from sitting one. S4 is round 2's paper and the builds of rounds 3 and 4. S5 is round 5. The paper decisions each slice needs are listed below and are unchanged. +`slice` S3 is round 1's attribution and round 2's measurement; its last part is the timings on a quiet machine. S4 is round 2's paper and gate, and the builds of rounds 3 and 4. S5 is round 5. The paper decisions S4's rebuild needs are taken; those of the history view and of S5 are listed below. How S2 was run, for the next slice. Tactical decisions 25 to 27 hold the rules. Two Fable designers with disjoint subjects took about twenty minutes. One workflow run of thirteen Opus agents then wrote the Specs, built four packages in sequence, bound the scenarios in four worktrees and integrated, in about an hour and a half. The main thread stopped and resumed that run once, at a seam between two packages, to add corrections: finished agents came back from the cache. The scripts, the rulings and the reports are in the S2 folder the project notes name. -What S2 leaves for the next opening: +What S2 left, as it stands now: -- The measured read count of `PlaceOrder`, 3N + 5 against the expected N + 6, is S3's first question. -- Sc L2-9's native observation, that the deployed function list equals a pinned list, needs one harness member: admin access cannot read `_system/cli/modules:apiSpec` through the harness, and an admin socket client can. Only the import lint is in place. -- A binding admits only the per-entity read model. The build that takes Sc L2-6 says how a history and an aggregate read model are bound and first activated. -- The first activation claims nothing about source events that already exist, and nothing stops it on a deployment with history. +- The measured read count of `PlaceOrder`, 3N + 5, has every term attributed (round 1) and stays the cost (round 2, C3). +- Sc L2-9's native observation of the deployed function list stays deferred: OD-021's lean takes the committed generated api as the pinned list. Its concurrency half is bound, the contention run. +- A binding admits only the per-entity read model. The build that takes Sc L2-6 says how a history and an aggregate read model are bound and installed. +- The first activation, which the code still runs, claims nothing about source events that already exist. The Specs replace it with the first rebuild, which 3b builds. Decisions each later slice needs on paper before its code: -- S4: how two projection versions are written at once, since a read model holds one projection and S2 built no version check (`projection-contract.sdp.md`, the open question C1 left); order-independent cross-stream folds and whether deletion keeps fold memory (`r3-architecture-history-fold-order-and-deletion`); who owns the tenant list (`r3-architecture-tenant-list-undefined`); operator authority and entry points (`r3-completeness-operator-entry-points-undefined`); refusal or a paused catch-up for history rollback (`r3-architecture-history-generation-rollback-unpaused`). -- S5: history-reading migrations off the command path unless the owner changes D3 (`r3-fidelity-migrate-on-load-replays-history-against-d3`); whether paused migrations are supported (`r3-fidelity-baseline-sweep-and-the-pause-contradict`); reopening authority, failure scope and drill cadence as explicit extensions (`r3-fidelity-restore-policies-cited-to-the-doc`); which diagnostic outcomes are emitted and where a duration comes from. +- S4, the history view (roadmap 4c): order-independent cross-stream folds and whether deletion keeps fold memory (`r3-architecture-history-fold-order-and-deletion`); the paused path's cursor across sources, its verify pass and its batch bound; the L2-6 interrupt example's "can roll back", which the rebuild Spec now answers with a refusal. The rest of S4's paper was taken in round 2: `design-s4-paper.md` sections 1 and 7. +- S5: history-reading migrations off the command path unless the owner changes D3 (`r3-fidelity-migrate-on-load-replays-history-against-d3`); whether paused migrations are supported (`r3-fidelity-baseline-sweep-and-the-pause-contradict`); reopening authority, failure scope and drill cadence as explicit extensions (`r3-fidelity-restore-policies-cited-to-the-doc`); which diagnostic outcomes are emitted and where a duration comes from; what a restore in place does with the schedules it keeps (`probe-7-architecture-restore-kept-schedules-not-reconciled`); how the restore check detects a stream state that is not the fold of its events (`s3-measurement-architecture-current-state-not-checked-against-event-fold`); the read-only comparison of the restore's read-model check (`s4-paper-sdp-restore-read-model-check-compare-mode-undefined`). What the platform review of 2026-10-02 asks of the later units. The ledger owns each finding's counterexample and what its build must show. This is the order and the guidance that fits no single finding. @@ -182,7 +208,7 @@ What the platform review of 2026-10-02 asks of the later units. The ledger owns ## Tactical decisions taken on 2026-10-01 -The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Any of them can be reopened. +The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words. Any of them can be reopened. 1. Slices replace the eight fix units of the previous plan. 2. A slice is built on Specs at `defined` with no open blocker inside the slice. The owner states `ready` after the slice passes. @@ -223,31 +249,39 @@ The owner accepted the first nine on the session's recommendation. The session t 37. The next roadmap is lighter. The owner's words on 2026-10-02: "next time, for the roadmap we will call only 1-2 advisors for input and give the decision for PM", then "25% weekly fable limits are better used for implementation", then "we should probably use one fable, one astra advisor for short-term roadmap stuff and fable PM to decide and main thread to rule with asking me for critical forks". So a roadmap run is one Fable advisor and one astra advisor for input, on the subjects where the roadmap is in doubt; the product advisor, a Fable agent, cuts the roadmap as its product manager; the main thread rules on the tactical questions; the owner is asked only for the critical forks. Fable is spent on the main thread and on the design of a slice on paper; forks are drafted by sol from the register's leans and read by an Opus agent, and an advisor rewrites one only when the owner names it. The four-memo run of this unit is the baseline the lighter run is judged against. 38. A round runs as one long orchestration, the way S1 and S2 ran and the guide's "Speed with assurance" says: one page of rulings in every brief, one Workflow run of Claude agents with GPT jobs started beside it, packages delegated and never findings, astra on the proof and the measurement, sol on bookkeeping and bounded builds, one integrator, one review behind the write lane, one pull request per lane with significant scope. The owner's words on 2026-10-02: "i would like to do a long orchestration based on notes made by fable orchestrating claude and gpt models. the approach enable significant scope in PRs." 39. A lane's worktree. Each lane of a round is a git worktree under `~/dev-libar/wt/` on its own branch from `main`, with `node_modules` a link to the main tree's and the backend binary named by `CONVEX_BACKEND_BINARY`, and no `.cache` link, because `.gitignore` ignores the directory and not a link to it. Every brief says: never run `npm ci` or `npm install` in a worktree. A link is never created over an existing link. The main thread merges `main` into a lane by hand where two lanes touched one line, and runs the gates before the pull request. +40. The owner delegated the forks of sitting one to the main thread on 2026-10-02 ("Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions.") and, when the register tool refused a session ruling on an `owner` row, chose the cleanest path: "i agree with the least noisy and cleanest path forward." So the twelve rows are recorded as the owner's, each ruling ending with both sentences, the tool is unchanged, and OD-048 stays open. +41. A round's lane is owned by an Opus lane lead from the brief to the last commit. It writes the GPT job's brief, starts the job with the companion from Bash, reads every line the job wrote, fixes what sits below the line itself, sends one numbered list of what sits above it back to the same job, and answers the main thread in fifteen lines. The main thread rules on the answers and merges. Nine lanes ran so in round 2; the project notes hold the numbers. +42. The owner's words of 2026-10-02 and 2026-10-03: "we will need a feature branch" and "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue". So a lane branches from `feature/round-2` and merges back into it, and nothing goes to `main` until the owner has reviewed the branch. Decisions 32 and 39 hold for the rest: a review before `main`, and a worktree per lane. And: "We can discuss details before preparation of work for the next session." +43. One Fable designer took the whole S4 paper, both subjects, because the gate, the rebuild and the operator entries share surfaces, and the owner said "fable only for key design decisions on a larger bit of roadmap". Opus leads, sol and astra did the rest. Fable was spent on the main thread and that one designer. +44. The ThinkPad (Fedora, an Intel i5 with 16 GB) is the development machine for now. The owner's words of 2026-10-03: "We will continue development of this project here on the thinkpad. It is not ideal, but sufficient. M3 Mac is continuting with macos native development." Seven lanes at once drove its load average to 13 to 18 and made native tests time out. So at most three or four lanes that start native backends run at once, a test that fails on a timeout is rerun once alone, and a latency or throughput timing is taken while nothing else runs. +45. An integrator never uses `pkill` on vitest: `pkill -f "vitest run"` matched every lane's run once and interrupted another lane's native file. A run is stopped by its own process ID. ## Owner queue -Every open decision is a row of `design/decisions/register.json`, and `python3 design/tools/decisions.py` filters it: `--owner` lists what waits for the owner, forks first, `--owner --before S4` what the owner must decide before S4, `--decided --by advisor --since ` what agents decided. The items that used to stand here are rows, each with its source and line; the register at the close: +Every open decision is a row of `design/decisions/register.json`, and `python3 design/tools/decisions.py` filters it: `--owner` lists what waits for the owner, forks first, `--owner --before S4` what the owner must decide before S4, `--decided --by advisor --since ` what agents decided. The register at the close: ``` -total: 153 -status: unsorted=0, sorted=109, waiting=27, decided=2, folded=15 -class: tactical=3, delegated=85, owner=50, null=15 -advisor: convex=35, domain=45, operator=34, product=39 +total: 154 +status: unsorted=0, sorted=100, waiting=24, decided=15, folded=15 +class: tactical=3, delegated=86, owner=50, null=15 +advisor: convex=36, domain=45, operator=34, product=39 ``` -Of the 50 `owner` rows, these block S3, S4 or S5, in the order the units need them. Thirteen forks exist, for sitting one: the eight that block S3 or S4, OD-047 and OD-048, and the three rows round 1 added, OD-151, OD-152 and OD-153. Sitting two takes the language rows, OD-008 with OD-006, OD-138 and the three that block S5; their forks are drafted the same way when the owner asks. +Decided since the last close: the twelve `owner` rows of sitting one, recorded as the owner's (decision 40), and OD-007, decided by the convex advisor with its checked lean as the ruling. OD-154, the grant read, is a new `delegated` row with its measured cost. + +Sitting two waits for the owner. Its forks are drafted the same way as sitting one's when the owner asks: -- `OD-046`, S3, the product advisor: The maximum order size and the latency and throughput targets, before S3 benchmarks. -- `OD-064`, S3, the convex advisor: OQ3: the largest order `PlaceOrder` supports is a product decision; the experiment measures with a provisional maximum o -- `OD-123`, S3, the domain advisor: OQ3: one document per stream as the default, and the largest order the placement command supports, are product decisions -- `OD-025`, S4, the operator advisor: Who creates a tenant's first grant. -- `OD-068`, S4, the domain advisor: Extension E-8: the doc names a generation registry, a marker and a gate without their shapes, and says to keep the old g -- `OD-069`, S4, the domain advisor: Extension E-8, reach of a generation: the doc says nothing about tenants and rebuild; the option taken here is one gener -- `OD-041`, S5, the domain advisor: D3 and a history-reading migration (`r3-fidelity-migrate-on-load-replays-history-against-d3`). -- `OD-042`, S5, the operator advisor: Restore policy: whether a failed check keeps every writer out with no override, and how often the drill runs (`r3-fideli -- `OD-114`, S5, the operator advisor: Probe 7 pending, the do-nothing check at activation: a plain scheduled mutation plus a scan of `_scheduled_functions` fo +- `OD-048`, the licence, the one fork of sitting one left the owner's. It blocks nothing before round 5c. +- `OD-044`, D2's premise, with F18 against it: on the pinned backend one transaction's rows share one commit timestamp in the parent and in components and later transactions get larger ones, while D2 and `spec:context.journal` say no order token for the whole deployment exists. Nothing uses the internal `getSnapshotTs()`. +- `OD-063`, OQ1, a component per context, with its local numbers on the lean: a fixture component call cost 3.56 and 5.76 ms over a helper call in two readings under load, and Probe 11 read 7.6 to 15.5 ms per empty component call, all at the native backend tier on the local backend. The quiet reading is still to come. +- `OD-114`, D13's do-nothing check, with Probe 7's facts: a restore in place keeps schedules the restored data no longer explains, a fresh restore keeps none, and a scan of `_scheduled_functions` stops at about 4,088 outstanding rows on the read limit. +- The language rows, OD-009, OD-022, OD-029, OD-139 and OD-140, with the words the round added: "snapshot" for the exported archive, which `CONTEXT.md` lists to avoid for state; "tenant fill" for `fillTenants`, `fillTenantsBatch` and `tenantFill`; the measurement's words (measurement record, cell, execution against attempt, rerun, contention, controlled change, local backend against hosted deployment); and the words of `ROADMAP.md`'s list. +- OD-008 with OD-006, OD-138, and the S5 rows OD-041 and OD-042, as round 3's sitting two cuts them. +- The two speed targets of F2 that are not yet compared, latency and throughput, because their timings wait for a quiet run (`[timings]` in "Where the work stands"). F2 says a target that is missed is reported as not held with its number and none is changed after the run. -The 85 `delegated` rows carry 32 leans in full. A `gpt-6.1-sol` check read the first 23: twelve held on every checked claim and carry `checkedBy`; eight hold with a gap and three do not hold (OD-066, OD-103, OD-109: a formula with no source, a blank citation and an unresolved Spec id, a `settles` that says built where the code is not), each waiting for its advisor's correction with the check's findings in the unit folder the project notes name; the nine written in the second round are unchecked. The check also found that most leans state facts without a label or a citation where the protocol wants one: the advisors' first run follows the sorting rule well and the rules of a sentence loosely. A checked lean is applied to the Specs as a ruling by a later unit, one Spec family at a time, and the row becomes `decided`. The rest carry a one-line lean or wait for a probe, a measurement or a build (`--waiting`). +`delegated` rows that wait for the next sort of their lens, each with the round's result on its lean: OD-057 (Probe 6 and the rebuild's own batch driver), OD-066 (3N + 5 with every term attributed; the journal tail read is 10 documents and 6,010 bytes, 29 percent of the documents read on ten lines), OD-120 (Probe 7's states) and OD-154 (3 documents and 918 bytes for the grant read) for the convex lens, and OD-081 (Probe 7's restore answers) for the operator lens. The session may not decide a `delegated` row (`decisions.py`: "session may decide only tactical rows"). + +The 86 `delegated` rows carry the leans of the first panel run. A `gpt-6.1-sol` check read the first 23: twelve held on every checked claim and carry `checkedBy`; eight hold with a gap and three do not hold (OD-066, OD-103, OD-109), each waiting for its advisor's correction with the check's findings in the unit folder the project notes name; the later leans are unchecked. A checked lean is applied to the Specs as a ruling by a later unit, one Spec family at a time, and the row becomes `decided`. The rest carry a one-line lean or wait for a probe, a measurement or a build (`--waiting`). ## Leads @@ -269,8 +303,10 @@ A session confirms or drops each. None is a finding yet. - Two acceptance examples require version equalities their contracts cannot produce. - History batch byte estimates leave out baseline events. - A fence does not limit how many batches are scheduled. + + The S4 paper of round 2 answers the four checked items and most of the per-entity ones in the rebuild and registry Specs: `design-s4-paper.md` section 2, "Lead 1", names the line for each, and what it leaves (the standalone baseline writer, the aggregate and history items) stays here. 2. Five `fix` fields in the ledger are wrong as written: a timestamp-only rebuild cursor skips ties; `floor(4 MiB / 64 KiB)` is 64 and payload size is not row cost; transaction metrics are `{ used, remaining }` behind an async call; Workpool 0.4.7 does not hand `onComplete` the final attempt count; a Node handler needs its own `"use node"` action file. -3. Two Convex facts no Spec covers. Nested calls draw on a system-operation time budget that byte and document bounds do not see: 605 calls used 14.8 s of a transaction that failed at 15.1 s on the earlier platform's native backend, in test mode, on 2026-09-26. And `convex-helpers` checks `maximumBytesRead` after it has read the row. +3. Two Convex facts no Spec covers. Nested calls draw on a system-operation time budget that byte and document bounds do not see: 605 calls used 14.8 s of a transaction that failed at 15.1 s on the earlier platform's native backend, in test mode, on 2026-09-26. And `convex-helpers` checks `maximumBytesRead` after it has read the row. Both are now facts probed on the pinned release, F19 and F20, from round 2. 4. The local backend refuses writes past 4 MiB a second. That bounds what slice S3 can measure on it, and a throughput number from it states the limit. 5. Convex ships `getConvexSize` and `getDocumentSize`. They answer the two byte-bound findings of S1. And `ctx.meta.getTransactionMetrics()` counts documents read and written across nested and component calls, in queries and mutations: S0 used it in two probes, and it is a cheap way for S1 to measure what a command reads. 6. Fifteen places where the design or a proposed fix repeats a shape the earlier platform regretted, each with the Spec line: progress kept on a row every command reads, work repeated per entry or per boundary, a stored maximum version taken as proof of coverage, a closed state with no route out, bookkeeping on the healthy path with no bounded lifetime, nested command calls where a helper would do, a recovery scan blocked by healthy work, registries with no reader. @@ -292,7 +328,7 @@ A session confirms or drops each. None is a finding yet. 20. Left by the read of S1's fold-in. Revocation reads every grant of a principal, so a principal with more than 500 grants whose fields are very large passes the transaction's read limit and cannot be revoked: a bounded, resumable revocation and a bound on a grant's field sizes would close it. Seven tests would not catch a removal: tenant routing in the revocation tests, the service-issuer test that never calls a public entry, the stock-overflow code in a declaration's list, grouping inside `placeOrders`, acceptance at exactly 8 MiB of measured writes, both receipt arrays filled at once, and the `invalidInput` path from a context through the closed code list. -21. Left by the review of S2 on purpose. `scripts/codegen.mjs` loses its first error when disposal also fails. Tests that would not catch a removal: one case in `tests/simulator/example.test.ts`; the Probe 5 test of a full first page passes even if the row cap never reaches `paginator`; nothing checks that the order DTO's `placedAt` equals the summary row's; a mutation that drops the composition in `scripts/codegen.mjs` is caught only by CI's codegen diff, and `scripts/dev.mjs` has no test. The production isolation rule checks relative imports and not bare specifiers. A total above the largest safe integer has no rejection code, and stream IDs inside a receipt's `affected` and `versions` have no length bound. The "no job" assertions of Sc L2-2 and L2-3 close a function-log window with an ordinary read, so a job scheduled with a delay past the window is not seen: the harness cannot read `_scheduled_functions`. A failed top-level mutation's completion record shows no written documents, so the function log cannot show rolled-back writes, and the first context's writes in Sc L2-1 are shown at the pure tier and by the retry. +21. Left by the review of S2 on purpose. `scripts/codegen.mjs` loses its first error when disposal also fails. Tests that would not catch a removal: one case in `tests/simulator/example.test.ts`; the Probe 5 test of a full first page passes even if the row cap never reaches `paginator`; nothing checks that the order DTO's `placedAt` equals the summary row's; a mutation that drops the composition in `scripts/codegen.mjs` is caught only by CI's codegen diff, and `scripts/dev.mjs` has no test. The production isolation rule checks relative imports and not bare specifiers. A total above the largest safe integer has no rejection code, and stream IDs inside a receipt's `affected` and `versions` have no length bound. The "no job" assertions of Sc L2-2 and L2-3 close a function-log window with an ordinary read, so a job scheduled with a delay past the window was not seen; round 2 added a read of `_scheduled_functions` beside the window. A failed top-level mutation's completion record shows no written documents, so the function log cannot show rolled-back writes, and the first context's writes in Sc L2-1 are shown at the pure tier and by the retry. 22. Temporal sentences still in Specs no S2 writer touched, beyond lead 16: `facts/probe-plan.sdp.md:38-40,74`, `decisions/d06` and `d08`, `facts/f11`, the first-run bullets of the Probe 1 to 5 examples, and "today's" in `platform/transactional-domain-platform.sdp.md`, `vocabulary.sdp.md`, `existing-systems.sdp.md` and `layers-and-profiles.sdp.md`. 23. Left by the platform review unit on purpose. Each is a hole in the handling of malformed input, a test gap, or a Spec line about what is not built. @@ -307,13 +343,26 @@ A session confirms or drops each. None is a finding yet. 25. From the platform review, not findings: - Pagination across a generation switch. Keep two subscribed pages pinned with both cursor endpoints, switch the active generation between reads, and run the native case for both ordinary and split pages. Rule and prove either continuity with no missing or duplicate subject or an explicit reset that makes the caller restart the page sequence; a cursor must not silently change its generation meaning. - Sharpen Leads 1's overlapping paused rebuilds. Two paused rebuilds share one source scope, and the first resumes or aborts while the second still needs the pause. The native case must show either refusal of the second owner before it changes anything or independent ownership whose release cannot reopen the other rebuild's writers; a bare scope-wide Boolean is not proof of ownership. - - Scheduled argument size. The current scheduling documentation says 8 MB for aggregate arguments while the limits page says 16 MiB. When the scheduling probe activates, resolve the discrepancy against the pinned backend with boundary cases and record which limit and accounting unit it actually enforces; do not turn either page into a confirmed defect now. + - Scheduled argument size. The scheduling documentation says 8 MB for aggregate arguments while the limits page says 16 MiB. Probe 7 resolved it on the pinned release: 16,777,216 bytes of Convex value bytes over one mutation's scheduled arguments, recorded on F13. 26. From the first run of the advisor panel. Eleven full leans wait for their advisor's correction (the check's file is `check-leans.json` in the unit folder); the nine leans of the second sort round have no check yet. The forks of sitting one were drafted in round 1 without an advisor. The advisors sort well by the rule and label their sentences loosely: a run of the panel after this one should start from the protocol's rules of a sentence and plant one false citation per lens to see whether the check catches it. The Convex advisor's lean on OD-044 reports that `getSnapshotTs`, which a ledger item relies on, is in no server declaration of `convex` 1.46.0; the commit timestamp is (`db.vars.commitTs`, `database.d.ts`). The advisors' language gaps (first activation, parent and context query, acceptance check, verifier, build tier, run record, gate, dispatch, lease, repair, drill, adopter, and three meanings of register) join the owner's ruling on terms, OD-139 and OD-140. 27. From the roadmap unit. `ROADMAP.md`, "Findings beyond the roadmap", lists seven things the advisors found beside their task: the order summary's status validator admits a `cancelled` no event sets; `@convex-dev/migrations` is not pinned; OD-048's `reason` cites a line of `docs/modern-ts.md` as `package.json`; the ledger's `files` fields mix two path prefixes; `CONTEXT.md` names three fixture names where the fixture has five; no harness member exports or imports a snapshot. Each names the unit that takes it. Three of the four questions became rows in round 1, OD-151 to OD-153; the fourth is OD-046 and OD-064 in plain words. The memos' words that `CONTEXT.md` lacks join OD-009 and OD-022. -28. From round 1. `design/specs/application/rebuild.sdp.md` near line 45 promises a one-write rollback and near line 84 a reverify and switch: the fork of OD-068 shows both, and S4's paper rules it. The pipeline accepts a stock stream row at version zero with positive stock and no event, which no command makes and the version-zero comparison planted. Whether the journal tail read, N documents and 601N bytes on the ten-line order, is worth the missing-tail and advanced-tail defects it catches, and whether the grant read narrows from four documents, are round 2a's questions, and removal needs the writer-invariant proof first. OD-066 is decidable: its formula is now 3N + 5 with every term attributed. `writeTable` of admin access was never exercised against a component table; the snapshot test needs an `unzip` binary nothing pins; after a temporary deploy of a fixture copy, `BackendFacts` still says `fixture`; `snapshot` is on the avoid list of `CONTEXT.md` and the restore Spec and the harness use it. The f15 remainder names a work stage and a run date, which overlaps the temporal sweep of leads 16 and 22. `design/PLAN.md` near lines 297 and 299 still lists one example for Sc L2-2 and L2-4. The harness's child runner now has a both-streams output whose timeout reports through the bounded error and not the "did not finish" message. +28. From round 1. `design/specs/application/rebuild.sdp.md` near line 45 promises a one-write rollback and near line 84 a reverify and switch: the fork of OD-068 shows both, and S4's paper rules it. The pipeline accepts a stock stream row at version zero with positive stock and no event, which no command makes and the version-zero comparison planted. The journal tail read and the grant read stay, by C3 and C4 of round 2, and removal needs the writer-invariant proof first. OD-066 waits for its lens with 3N + 5 attributed. Probe 7 closed three items of this lead: `writeTable` is exercised against component tables, a ZIP reader replaced `unzip`, and `BackendFacts.composition` is `null` after a temporary deploy. `snapshot` is on the avoid list of `CONTEXT.md` and the restore Spec and the harness use it. The f15 remainder names a work stage and a run date, which overlaps the temporal sweep of leads 16 and 22. `design/PLAN.md` near lines 297 and 299 still lists one example for Sc L2-2 and L2-4. The harness's child runner now has a both-streams output whose timeout reports through the bounded error and not the "did not finish" message. + +29. From round 2. Each is a test gap, a hole in a failure path, a Spec line about what is not built, or a number still to take. + - `design/tools/test_decisions.py` has one test that fails at `main` too: `test_real_register_has_unique_rows_and_exact_recipe_20_coverage` ends in an `IndexError`. It is not one of the gates. + - Journal inspection. `orders-inventory-example.sdp.md` near line 70 names the context queries that return a stream's events and its current version; the Orders and Inventory contexts register only `get` and `list`, no `history` query of `spec:context.queries`, and the native tests read journals through admin table reads. The Spec line stays as intended truth. + - The Probe 6 context callback at `fixture/convex/migrations.ts:198-203` writes a `draft` summary beside the source's version whatever the source's status. Fixture probe code that nothing rests on; fixed only if that fixture is reused. + - A partial tenant list passes `startGeneration`'s refusal: a list that holds a tenant whose first grant came after the table, and misses an older one, gets through. The rebuild Spec states it as an open question and tells the operator to run `fillTenants` first. `limitTenantFillBatch` is 8 because a grant row is bounded only by the 1 MiB document limit. + - An event payload above the 16,384 bytes `append` holds it to reaches the caller as a bare technical failure: `append` throws a plain error inside the context, and nothing is stored. The example's stock item ID bound now refuses the case that reached it first; any command whose payload can outgrow the bound meets the same failure. + - The timings not yet taken on a quiet machine. `PlaceOrder`'s latency and throughput: `npm run measure:place-order -- --quiet`, from `record-s3-measurement.md`. F19's nested and component calls: the command in `record-fact-rows.md`, whose times go into F19's `nativeMeasurements`. + - `first-experiment.sdp.md` `observedPlaceOrderDocuments` says "no gate read". Every command now reads the gate, and with no gate document the read counts no document, so the six native files that count reads pass unchanged: the bullet should say "no gate document". + - From the gate: a duplicate is answered from its receipt in a closed scope and under the restore door, as the Spec now says; the plain error for an audited declaration with no subject has a built message and no pinned one; `baseline-operations.broken-audit-aborts` stays unbound until roadmap 4a builds an audited command. + - From `CancelOrder`: an invariant that exempts a cancelled order from its time and total checks survives every test; the bounds of the two context operations, Orders' 32 streams and Inventory's 100, have no direct `convex-test` case; `CancelOrder` declares no `bounds`, so a long order ID is refused by the adapter at 256 bytes after authorization and the receipt read. + - From the S4 paper: E-48 and E-49 are numbers of package D's range on a package C Spec, because C's range is full; the journal's baseline driver is not written as an operator entry, and the baseline writer changes sources without maintaining read models; the receipts sweep loops with `nextTenant`, which `receipt-table.sdp.md` does not say yet; a scheduled mutation that loses a race is assumed to rerun until it commits, and no pinned package shows a bound. + - From the probes, not run: Probe 6's timeout, retry exhaustion, a restore with a component worker pending and `parallelize`; Probe 7's hosted dashboard restore, the seven-day expiry, Workpool and Workflow (OD-115), file storage, a schema change between export and import, the Layer 2 restore drill, the argument limits of an action, and the limit of one million outstanding schedules. The reports behind leads 1 to 3 and 6 are outside the repository, because they quote a private repository. The project notes file named in `AGENTS.md` lists them, with the reports, briefs and rulings of S0. diff --git a/docs/sdp-feedback.md b/docs/sdp-feedback.md index 3238b96..e9ba105 100644 --- a/docs/sdp-feedback.md +++ b/docs/sdp-feedback.md @@ -4,6 +4,10 @@ A running file, addressed to the Protocol's maintainers. Each `adopt` unit and e Every number here was measured on this corpus on the date of its section. Re-run before relying on one. +## 2026-10-03, round 2 of the roadmap + +- A model Spec now owns an example space: `orders-inventory-example.sdp.md` holds the `gwt-vocabulary` of its six examples, and the pinned Protocol validates it with 0 errors and 0 warnings, while the model template of this corpus's `design/PLAN.md` 6.1 lists only Intent and Model, so nothing told the author it was allowed. + ## 2026-10-01, slice S0, the first build on the Protocol at `5993da7` ### A Spec with designed stubs is a maturity the Protocol cannot state From ba37ad98e4a861996b0870e12d46183042949aec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:13:55 +0200 Subject: [PATCH 065/155] build: getGateAudit passes only the cursor and the capped count, so an end cursor cannot widen a page Co-Authored-By: Claude Opus 5.5 --- design/specs/application/write-pause.sdp.md | 2 +- src/gate/entries.ts | 3 +- tests/simulator/gate.test.ts | 35 +++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index 165f41d..a5450d0 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -73,7 +73,7 @@ The gate is one document so that a command pays one point read. Scopes are strin - fnCloseGate: `export const closeGate = internalMutation({ args: { scopeKey: v.string(), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `closeScope` with no generation: it calls `assertOperator`, throws `"closeGate takes the scope all or a tenant scope, and " + scopeKey + " is neither"` unless `scopeKey` is `all` or `tenant:` followed by a tenant ID of 1 to `limitIdLength` bytes, throws `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and then calls `closeScope`; it does not read the tenant list, so a tenant may be closed before its first grant; this is also how a meaning migration's sweep is run under a pause (E-8, E-25, E-48, D9, D19) - fnResumeGate: `export const resumeGate = internalMutation({ args: { scopeKey: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `resumeScope`: it calls `assertOperator`, makes the scope check of `closeGate` with `resumeGate` in the message, so a source scope is never reopened ahead of its rebuild, and then calls `resumeScope` (E-8, E-48, D9, Sc L2-6) - fnGetGate: `export const getGate = internalQuery({ args: {}, returns: v.object({ restore: v.boolean(), closed: v.array(closedEntryValidator) }), handler })` answering `restore` true when `MAINTENANCE_MODE` is `restore` and `closed` as the document holds it, empty when there is no document (E-8, E-42, D19) -- fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page, a request for more answered with a page of 100, where `operatorAuditDoc` is the `operatorAudit` table's document validator with `_id` and `_creationTime` (E-8, E-43, D19) +- fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page, a request for more answered with a page of 100, passing the paginator only the request's `cursor` and the capped `numItems`, so an `endCursor` in the request is ignored and cannot widen a page, where `operatorAuditDoc` is the `operatorAudit` table's document validator with `_id` and `_creationTime` (E-8, E-43, D19) - occSerialization: a writer's mutation reads the gate document, or the index range that holds none; a close that commits first invalidates that read, the writer re-runs, reads the closed gate and refuses; a writer that commits first wrote before the close, which is the consistent cut; a batch scheduled before the close but started after it reads the closed gate like any writer, so no closing state, drain interval or confirmation is needed, and the one-second mutation timeout plays no part (F1, D9, E-8) - observedRestoreDoorWithoutDeploy: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, a change of `MAINTENANCE_MODE` through the admin API's update of environment variables is seen by the next function run with no deploy: set to `restore`, the next command in every tenant, through the public and the internal entry, is refused with `"write paused for all: restore"` and `getGate` answers `restore` true; set to another value, `Restore` among them, the refused request keys apply; `spec:application.write-pause.restore-door-without-deploy` binds it at the native tier, and it is read again when the pin moves (E-42, F12) - limitClosedScopes: at most 16 scopes closed at once, so the gate document stays a few kilobytes and the writer check scans an array in memory (E-8, F13) diff --git a/src/gate/entries.ts b/src/gate/entries.ts index c838b67..1af0d53 100644 --- a/src/gate/entries.ts +++ b/src/gate/entries.ts @@ -98,8 +98,9 @@ export const getGateAudit = internalQuery({ .query("operatorAudit") .withIndex("by_scope", (q) => q.eq("scopeKey", scopeKey)) .order("desc") + // Only the cursor and the capped count: an end cursor would let one page pass the bound. .paginate({ - ...paginationOpts, + cursor: paginationOpts.cursor, numItems: Math.min(paginationOpts.numItems, limitOperatorQuery), }), }); diff --git a/tests/simulator/gate.test.ts b/tests/simulator/gate.test.ts index 23a5d69..bb0ba69 100644 --- a/tests/simulator/gate.test.ts +++ b/tests/simulator/gate.test.ts @@ -500,6 +500,41 @@ test("convex-test: getGateAudit pages one scope newest first with at most 100 re expect(small.page).toEqual(expected.slice(0, 1)); }); +test("convex-test: getGateAudit ignores an end cursor, so a page past the first two still holds at most 100 records", async () => { + const t = gateApp(); + await t.run(async (ctx) => { + for (let i = 0; i < 205; i++) + await ctx.db.insert("operatorAudit", { + kind: "gate.close", + scopeKey: "all", + reason: `reason-${i}`, + operator: "op", + recordedAt: i, + }); + }); + const page = (cursor: string | null, endCursor?: string) => + t.query(internal.gate.getGateAudit, { + scopeKey: "all", + paginationOpts: { + numItems: 100, + cursor, + ...(endCursor === undefined ? {} : { endCursor }), + }, + }); + const first = await page(null); + const second = await page(first.continueCursor); + expect([first.page.length, second.page.length]).toEqual([100, 100]); + const ended = await t.query(internal.gate.getGateAudit, { + scopeKey: "all", + paginationOpts: { + numItems: 1, + cursor: null, + endCursor: second.continueCursor, + }, + }); + expect(ended.page).toEqual(first.page.slice(0, 1)); +}); + // spec:application.write-pause fnCloseScope; spec:operations.baseline-operations tableOperatorAudit, fnWriteOperatorAudit. test("convex-test: closeScope copies an existing generation into the entry and its audit", async () => { const t = gateApp(); From a309619f351f501daaab18d642fd1d38e7ef9fbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:14:48 +0200 Subject: [PATCH 066/155] build: insertGrant refuses a tenant ID above limitIdLength as invalidInput Co-Authored-By: Claude Opus 5.5 --- design/specs/command/actor-and-scope.sdp.md | 2 +- src/command/authority.ts | 10 ++++++++ tests/simulator/tenants.test.ts | 26 +++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/design/specs/command/actor-and-scope.sdp.md b/design/specs/command/actor-and-scope.sdp.md index a1cd43e..49d8b3c 100644 --- a/design/specs/command/actor-and-scope.sdp.md +++ b/design/specs/command/actor-and-scope.sdp.md @@ -74,7 +74,7 @@ This contract pins the shapes the parent establishes and passes into components: - fnEstablishActor: `establishActor(ctx: MutationCtx | QueryCtx, serviceIssuers: ReadonlySet): Promise` returns `null` when `ctx.auth.getUserIdentity()` is `null`, a `service` actor when the identity's issuer is in `serviceIssuers`, and a `human` actor otherwise, with `id` from `tokenIdentifier` (D11, E-37) - typeQueryPolicy: `type QueryPolicy = { name: string; tenantId: string; permission: string; subject?: SubjectRef }` what a parent query states before it discloses anything: its own name, the tenant it reads and the permission the read requires (D11, Law 5, E-6) - fnAuthorizeQuery: `authorizeQuery(ctx: GenericQueryCtx, policy: QueryPolicy, serviceIssuers?: ReadonlySet): Promise` calls `establishActor` and then `authorize` for the policy's tenant, permission and subject, as the pipeline's steps 2 and 4 do, and returns the actor; it throws through `reject` with code `unauthenticated` when there is no identity and `forbidden` with `details.reason` when the decision denies, each with the query's `name` where a command's type goes; it writes nothing, and it is generic over the parent's data model because a database reader typed with the parent's tables is not assignable to one typed with the library's (D11, Law 5, D7, E-6) -- fnInsertGrant: `insertGrant(ctx: { db: MutationCtx["db"] }, grant: GrantInput): Promise>` first calls `assertWritable(ctx, scopesOfUseCase(grant.tenantId, []))` of `spec:application.write-pause`, so a closed `all` or tenant scope, or the restore door, refuses it with `writePaused` before it reads anything, then reads `tenants` by `by_tenant` for `grant.tenantId` and inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, then inserts one grant row with `grantedAt` the mutation's `Date.now()`, where `GrantInput = { tenantId: string; principalKind: ActorKind; principalId: string; permission: string; subject?: SubjectRef; grantedBy: string }`; a plain helper the library exports and registers no function for (D11, E-37, E-49, E-8) +- fnInsertGrant: `insertGrant(ctx: { db: MutationCtx["db"] }, grant: GrantInput): Promise>` first refuses a `grant.tenantId` of more than `limitIdLength` bytes of UTF-8 with the rejection `invalidInput`, the `ConvexError` data `{ code: "invalidInput", message: "tenantId has at most 256 bytes of UTF-8", details: { field: "tenantId", length, limit: 256 } }`, before any read, then calls `assertWritable(ctx, scopesOfUseCase(grant.tenantId, []))` of `spec:application.write-pause`, so a closed `all` or tenant scope, or the restore door, refuses it with `writePaused` before it reads anything, then reads `tenants` by `by_tenant` for `grant.tenantId` and inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, then inserts one grant row with `grantedAt` the mutation's `Date.now()`, where `GrantInput = { tenantId: string; principalKind: ActorKind; principalId: string; permission: string; subject?: SubjectRef; grantedBy: string }`; a plain helper the library exports and registers no function for (D11, E-37, E-49, E-8) - fnRevokeGrant: `revokeGrant(ctx: { db: MutationCtx["db"] }, grant: Omit): Promise` makes the gate read of `insertGrant` first, then deletes the rows that grant this permission to this principal, on the named subject when one is given and otherwise on any, and returns how many it deleted; revocation is that delete (D11, E-37, E-8) - grantWriters: the application registers its own internal mutations over `insertGrant` and `revokeGrant`; a tenant's first grant is created with admin access through that internal mutation, and later grants the same way or by whatever command the composition registers; the library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation, which are an adopter's to build on the helpers; both helpers read the maintenance gate first, so the composition's grant mutations are held by a closed tenant, a closed deployment and the restore door like any writer; a native test creates its grants as setup with the harness's admin access (D11, E-37, E-15) - tableGrants: `grants: defineTable({ tenantId: v.string(), principalKind: actorKindValidator, principalId: v.string(), permission: v.string(), subject: v.optional(v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() })), grantedBy: v.string(), grantedAt: v.number() }).index("by_principal", ["tenantId", "principalKind", "principalId"]).index("by_permission", ["tenantId", "permission"])` (D11, Law 11, E-6) diff --git a/src/command/authority.ts b/src/command/authority.ts index ddb9629..55febd5 100644 --- a/src/command/authority.ts +++ b/src/command/authority.ts @@ -4,6 +4,8 @@ // operator's function uses to write and delete a grant, and nextTenant, which reads the tenant list. // None is registered here. import type { GenericDatabaseReader, GenericQueryCtx } from "convex/server"; +import { ConvexError } from "convex/values"; +import { limitIdLength, utf8Length } from "../context/text.js"; import { assertWritable, scopesOfUseCase } from "../gate/gate.js"; import type { GateDataModel } from "../gate/tables.js"; import type { Actor, AuthorizeInput, SubjectRef } from "./actor-and-scope.js"; @@ -105,6 +107,14 @@ export async function insertGrant( ctx: { db: MutationCtx["db"] }, grant: GrantInput, ): Promise { + // The tenant ID is stored in the tenant list, which holds at most limitIdLength bytes of it. + const length = utf8Length(grant.tenantId); + if (length > limitIdLength) + throw new ConvexError({ + code: "invalidInput", + message: `tenantId has at most ${limitIdLength} bytes of UTF-8`, + details: { field: "tenantId", length, limit: limitIdLength }, + }); await assertGrantWritable(ctx, grant.tenantId); const now = Date.now(); const tenant = await ctx.db diff --git a/tests/simulator/tenants.test.ts b/tests/simulator/tenants.test.ts index 2dc0ab9..b23e68d 100644 --- a/tests/simulator/tenants.test.ts +++ b/tests/simulator/tenants.test.ts @@ -1,4 +1,5 @@ import { convexTest } from "convex-test"; +import { ConvexError, type Value } from "convex/values"; import { afterEach, beforeEach, expect, test, vi } from "vitest"; import schema from "../../fixture/convex/schema.js"; import { @@ -78,3 +79,28 @@ test("convex-test: nextTenant reads one tenant in ascending ID order, excludes t "tenant:source:depot", ); }); + +// spec:command.actor-and-scope fnInsertGrant, tableTenants: at most limitIdLength bytes of tenant ID. +test("convex-test: insertGrant refuses a tenant ID of 257 bytes as invalidInput and stores neither a tenant nor a grant, and takes one of 256", async () => { + const t = app(); + const error = await t + .run((ctx) => insertGrant(ctx, grant("t".repeat(257)))) + .then( + () => null, + (thrown: unknown) => thrown, + ); + expect(error).toBeInstanceOf(ConvexError); + expect((error as ConvexError).data).toEqual({ + code: "invalidInput", + message: "tenantId has at most 256 bytes of UTF-8", + details: { field: "tenantId", length: 257, limit: 256 }, + }); + expect(await t.run((ctx) => ctx.db.query("tenants").collect())).toEqual([]); + expect(await t.run((ctx) => ctx.db.query("grants").collect())).toEqual([]); + await t.run((ctx) => insertGrant(ctx, grant("é".repeat(128)))); + expect( + (await t.run((ctx) => ctx.db.query("tenants").collect())).map( + (row) => row.tenantId, + ), + ).toEqual(["é".repeat(128)]); +}); From 301e36778a16bd81537de5294731c226d3d7720e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:15:08 +0200 Subject: [PATCH 067/155] design: fillTenants reads the restore door alone and is refused while it is closed Co-Authored-By: Claude Opus 5.5 --- design/specs/application/rebuild.sdp.md | 2 +- design/specs/application/write-pause.sdp.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 451d9e5..5ea06ad 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -136,7 +136,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged; resumeGeneration continues it"` when `pass` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `pass: "purge"`, `cursor: null` and no `lastError`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) - fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(v.object({ generation: generationDoc, progress: progressDoc })), handler })` the newest `limitGenerationsListed` generations of the read model by `by_read_model` in descending order, each with its progress row, where `generationDoc` and `progressDoc` are the two tables' document validators with `_id` and `_creationTime`; an operator entry that only reads, so it takes no operator (E-8, E-40, E-48, D19) - tableTenantFill: `tenantFill: defineTable({ pass: v.union(v.literal("fill"), v.literal("idle")), fence: v.number(), cursor: v.union(v.string(), v.null()), batchesDone: v.number(), tenantsRead: v.number(), tenantsInserted: v.number(), lastError: v.optional(v.string()), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), updatedAt: v.number() })` in `readModelTables`, at most one row per deployment, the progress row of the tenant fill and nothing else's; operational data of the deployment with no tenant dimension and no index, read with `ctx.db.query("tenantFill").first()`; `cursor` is the last tenant ID the walk passed, `null` before its first, `tenantsRead` counts the tenants the walk found and `tenantsInserted` the rows it inserted (E-49, E-40, E-48) -- fnFillTenants: `export const fillTenants = internalMutation({ args: { operator: v.string() }, returns: v.null(), handler })` in order: `assertOperator`; for each target of `config.targets` the in-flight check by `by_read_model_state`, throwing `"Read model " + name + " has generation " + n + " in " + state + "; fillTenants needs no generation in flight"`; then, with `now` the mutation's `Date.now()`, when no `tenantFill` row exists it inserts one with `pass: "fill"`, `fence: 1`, `cursor: null`, every count 0, `startedAt`, `changedAt` and `updatedAt` at `now` and `startedBy` and `changedBy` the operator, and when one exists it patches it to `pass: "fill"`, fence plus 1, the cursor and the counts kept when the pass was `fill` and `null` and 0 when it was `idle`, no `lastError`, `changedAt`, `changedBy` and `updatedAt`; then it schedules `refs.fillTenantsBatch` with the row's fence through `ctx.scheduler.runAfter(0, ...)`; so a call continues a fill whose chain a restore, an error or a closed gate ended, and a call after a finished fill walks the grants again and inserts only what is missing (E-49, E-48, E-40, F8) +- fnFillTenants: `export const fillTenants = internalMutation({ args: { operator: v.string() }, returns: v.null(), handler })` in order: `assertOperator`; the restore door, throwing `"fillTenants is refused while the restore door is closed"` when `process.env.MAINTENANCE_MODE` is exactly `restore`, because the fill writes the tenant list that a restore replaces, and no other read of the gate, because the fill writes no tenant's data and so no closed tenant scope stops it; for each target of `config.targets` the in-flight check by `by_read_model_state`, throwing `"Read model " + name + " has generation " + n + " in " + state + "; fillTenants needs no generation in flight"`; then, with `now` the mutation's `Date.now()`, when no `tenantFill` row exists it inserts one with `pass: "fill"`, `fence: 1`, `cursor: null`, every count 0, `startedAt`, `changedAt` and `updatedAt` at `now` and `startedBy` and `changedBy` the operator, and when one exists it patches it to `pass: "fill"`, fence plus 1, the cursor and the counts kept when the pass was `fill` and `null` and 0 when it was `idle`, no `lastError`, `changedAt`, `changedBy` and `updatedAt`; then it schedules `refs.fillTenantsBatch` with the row's fence through `ctx.scheduler.runAfter(0, ...)`; so a call continues a fill whose chain a restore, an error or a closed gate ended, and a call after a finished fill walks the grants again and inserts only what is missing (E-49, E-48, E-40, F8) - fnFillTenantsBatch: `export const fillTenantsBatch = internalMutation({ args: { fence: v.number() }, returns: v.null(), handler })` (E-49, E-40) - fillSteps: `fillTenantsBatch` loads the `tenantFill` row and returns without writing when it is absent, `pass` is not `fill` or `fence` differs from its argument; it calls `gateAllows(ctx, ["all"])` and, when the answer is not allowed, parks as step 3 does, patching `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt` and scheduling nothing, so the fill waits for `fillTenants` after the gate reopens; otherwise it repeats at most `limitTenantFillBatch` times: it reads `ctx.db.query("grants").withIndex("by_principal", (q) => (cursor === null ? q : q.gt("tenantId", cursor))).first()`, stops when that answers no row, and otherwise reads `tenants` by `by_tenant` for the grant's `tenantId`, inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, adds one to `tenantsRead` and, for an insert, one to `tenantsInserted`, and sets the cursor to that `tenantId`; a batch that took `limitTenantFillBatch` tenants patches the row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt` and schedules `refs.fillTenantsBatch` with the same fence; a batch that found no further grant patches the row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt` and schedules nothing (E-49, E-40, F8, Law 11) - limitTenantFillBatch: 8 tenants per fill batch; a grant row's bytes are bounded only by the 1 MiB document limit, so a batch reads at most 8 grant rows, 8 MiB, one per tenant through `by_principal`, at most 8 tenant rows of at most `limitIdLength` bytes of tenant ID each, the gate document and the `tenantFill` row, and writes at most 8 tenant rows and the `tenantFill` row, under the 16 MiB read and write ceilings (E-49, E-40, F13) diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index a5450d0..cfca537 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -37,7 +37,7 @@ The write pause is the maintenance gate every writer in a scope reads inside its ## Contract -- There is one gate document per deployment, read through `assertWritable`, which throws, by the command pipeline at its step 7 and by the grant helpers `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` before they write, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches, its tenant fill's `fillTenantsBatch`, and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states; every command that reaches step 7 reads the gate; a duplicate answered from its receipt does not, because its answer writes nothing (D9, D6, E-8, E-25, E-37, E-42) +- There is one gate document per deployment, read through `assertWritable`, which throws, by the command pipeline at its step 7 and by the grant helpers `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` before they write, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches, its tenant fill's `fillTenantsBatch`, and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries, the rebuild's `fillTenants`, which reads the restore door alone and is refused while it is closed, and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states; every command that reaches step 7 reads the gate; a duplicate answered from its receipt does not, because its answer writes nothing (D9, D6, E-8, E-25, E-37, E-42) - A scope is `all`, one tenant, or one source stream type of one context; the gate holds the set of closed scopes, each with a reason, the generation it serves if any, and the operator who closed it (E-8, E-48, D9) - A writer in a closed scope refuses before any write with the transient refusal `writePaused`, whose message carries the scope and the reason; nothing is stored, no operation ID is minted, and the same request key succeeds once the scope is open (D9, D4, Law 6) - A use case is in scope when `all` is closed, when its tenant is closed, or when a stream type its declaration's `writes` names is closed (E-8, E-38, D9) From a464abfca270f75214c7f08f11f0811fde10be3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:17:22 +0200 Subject: [PATCH 068/155] test: the measurement holds its duplicate count, its engine reruns and its summed usage against the records Co-Authored-By: Claude Opus 5.5 --- tests/native/place-order-measurement.test.ts | 14 +++++++++++++- tests/native/place-order-measurement.ts | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/native/place-order-measurement.test.ts b/tests/native/place-order-measurement.test.ts index b92bdcc..0d5891b 100644 --- a/tests/native/place-order-measurement.test.ts +++ b/tests/native/place-order-measurement.test.ts @@ -144,11 +144,23 @@ test.each([ : ([32, 0, 13106, 0] as const); expectUsage(required(result.records[0], "completion"), expected); expect(result.after).toEqual(result.before); - if (path === "authorized duplicate") + if (path === "authorized duplicate") { expect(result.answers[0]).toMatchObject({ kind: "applied", replayed: true, }); + // The answer from the receipt is the cell's one duplicate and no applied command. + expect(result.value.outcomeCounts).toEqual({ + applied: 0, + duplicates: 1, + rejected: 0, + businessFailure: 0, + technicalFailure: 0, + transientRefusal: 0, + }); + } + // One command and no concurrency: the engine reruns nothing. + expect(result.value.engineReruns).toBe(0); if (path === "changed input conflict") expect(result.answers[0]).toMatchObject({ kind: "rejection", diff --git a/tests/native/place-order-measurement.ts b/tests/native/place-order-measurement.ts index 5ecdb81..be300c3 100644 --- a/tests/native/place-order-measurement.ts +++ b/tests/native/place-order-measurement.ts @@ -342,6 +342,24 @@ export function recordMeasurement( measure("Measurement", json(value)); // A terminal record per submitted command is also the log-completeness check. expect(completed).toHaveLength(options.answers.length); + // Each reported count against a second reading of the same answers and records: a duplicate is an + // answer marked replayed, every execution that is not a command's last is an engine rerun, and the + // summed usage covers every execution, reruns included. + expect(value.outcomeCounts.duplicates).toBe( + options.answers.filter((answer) => answer.replayed === true).length, + ); + expect(value.engineReruns).toBe(value.executions - value.submittedCommands); + expect(value.engineReruns).toBe( + requests.reduce((sum, request) => sum + request.engineReruns, 0), + ); + for (const key of usageKeys) { + expect(value.usageStats[key]).toBe( + own.reduce((sum, record) => sum + usageOf(record)[key], 0), + ); + expect(value.usageStats[key]).toBe( + value.finalUsageStats[key] + value.rerunUsageStats[key], + ); + } expect(requests).toHaveLength(options.answers.length); expect(new Set(own.map((record) => record.executionId)).size).toBe( own.length, From e455cff553036151a6663ac7c4d7c0f754ee9cd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:17:44 +0200 Subject: [PATCH 069/155] test: PlaceOrder and ReceiveStock bound a decomposed stock item ID by the bytes supplied Co-Authored-By: Claude Opus 5.5 --- tests/pure/stock-item-id-bound.test.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/pure/stock-item-id-bound.test.ts b/tests/pure/stock-item-id-bound.test.ts index f740cf0..7512d77 100644 --- a/tests/pure/stock-item-id-bound.test.ts +++ b/tests/pure/stock-item-id-bound.test.ts @@ -168,3 +168,18 @@ test("pure: ReceiveStock refuses the first stock item ID past the same bound", ( ), ).toEqual(refusal(1, 65)); }); + +test("pure: both commands bound the supplied bytes, so a decomposed ID of 96 bytes is refused although its composed form has 64", () => { + const decomposed = "é".repeat(32); + expect(new TextEncoder().encode(decomposed)).toHaveLength(96); + expect(new TextEncoder().encode(decomposed.normalize("NFC"))).toHaveLength( + 64, + ); + const refineReceive = receiveStockDeclaration.refine; + if (refineReceive === undefined) + throw new Error("ReceiveStock needs its refinement"); + expect(refine([decomposed])).toEqual(refusal(0, 96)); + expect( + refineReceive({ items: [{ stockItemId: decomposed, quantity: 1 }] }), + ).toEqual(refusal(0, 96)); +}); From 411174980dd78eed6be49255c263104bec8ff0c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:20:06 +0200 Subject: [PATCH 070/155] test: the failed-schedule scans leave out another function's failed run, and the restore comparisons hold the tenant, gate and audit tables Co-Authored-By: Claude Opus 5.5 --- tests/native/scheduler-composition.ts | 4 ++++ tests/native/scheduler-functions.ts | 13 +++++++++++ tests/native/scheduler-scan.test.ts | 33 +++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 tests/native/scheduler-scan.test.ts diff --git a/tests/native/scheduler-composition.ts b/tests/native/scheduler-composition.ts index 925882b..edd8b5f 100644 --- a/tests/native/scheduler-composition.ts +++ b/tests/native/scheduler-composition.ts @@ -111,6 +111,10 @@ export async function dataRows(backend: Backend) { ? [ "receipts", "grants", + "tenants", + "maintenanceGates", + "auditRecords", + "operatorAudit", "generations", "orderSummaries", "schedulerData", diff --git a/tests/native/scheduler-functions.ts b/tests/native/scheduler-functions.ts index c8dd5ab..b562c04 100644 --- a/tests/native/scheduler-functions.ts +++ b/tests/native/scheduler-functions.ts @@ -76,6 +76,19 @@ export const reference = mutation({ return row; }, }); +// A scheduled run of a function other than reaction that fails, which the failed-reaction scans leave out. +export const refused = mutation({ + args: {}, + handler: () => { + throw new Error("refused: a function other than reaction"); + }, +}); +export const scheduleRefused = mutation({ + args: {}, + handler: async (ctx) => { + await ctx.scheduler.runAfter(0, mutationRef("refused"), {}); + }, +}); export const consume = mutation({ args: { payload: v.any() }, handler: () => null, diff --git a/tests/native/scheduler-scan.test.ts b/tests/native/scheduler-scan.test.ts new file mode 100644 index 0000000..e15bb4e --- /dev/null +++ b/tests/native/scheduler-scan.test.ts @@ -0,0 +1,33 @@ +import { expect, test, vi } from "vitest"; +import { schedulingBackend } from "./scheduler-composition.js"; +// The failed-reaction scan of the Probe 7 scan example, on the temporary scheduling composition: a +// failed run of another function sits in the same system table and is left out of both scans. +vi.setConfig({ testTimeout: 300000 }); + +test("native: the failed-schedule scans return the failed reaction and leave out a failed run of another function", async () => { + const { backend } = await schedulingBackend(); + await backend.admin.run("scheduling:states", { + label: "local", + due: Date.now() + 3600000, + }); + await backend.admin.run("scheduling:scheduleRefused"); + const failed = async () => + (await backend.admin.readTable("_scheduled_functions")) + .filter((row) => (row.state as { kind: string }).kind === "failed") + .map((row) => row.name) + .sort(); + await expect + .poll(failed, { timeout: 30000 }) + .toEqual(["scheduling.js:reaction", "scheduling.js:refused"]); + const { rows } = (await backend.admin.run("scheduling:scan")) as { + rows: { name: string; args: { label: string }[] }[]; + }; + const plain = (await backend.admin.run("scheduling:scanPlain")) as { + name: string; + args: { label: string }[]; + }[]; + for (const answer of [rows, plain]) + expect(answer.map((row) => [row.name, row.args[0]?.label])).toEqual([ + ["scheduling.js:reaction", "local-failed"], + ]); +}); From a7459161b0e238f5bcc87c04c0b1355177f7b204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:21:45 +0200 Subject: [PATCH 071/155] design: the first experiment and F19 take the quiet-machine timings with their pins Co-Authored-By: Claude Opus 5.5 --- design/specs/application/first-experiment.sdp.md | 2 ++ .../specs/facts/f19-nested-calls-share-time-budgets.sdp.md | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 2c6a84c..1cc5a91 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -122,6 +122,8 @@ The experiment is an application, a test suite and a report. The application is - observedMeasurementBytes: [extension] at the native backend tier with `observedMeasurementPins`, fixed seven-character stock and order IDs, four grants and one active generation, orders of 1, 10 and 100 distinct stock items received once read 2943, 14454 and 129564 bytes and write 4271, 18410 and 159800 bytes; the corresponding write-index rows are 25, 97 and 817; a ten-line authorized duplicate or changed-input conflict reads 5 documents and 3441 bytes and writes zero, while short stock or a missing journal tail on the last line reads 32 documents and 13106 bytes and writes zero (E-47, F13) - observedContentionCosts: [extension] at the native backend tier with `observedMeasurementPins`, each of the nine cells at 1, 10 and 100 lines and 2, 8 and 32 callers reached one applied outcome and all remaining short-stock rejections with zero engine failures; an execution refused at commit read the healthy path's documents and bytes and wrote zero, and its final short-stock rejection, with the shared item first, read 5 documents and 1595 bytes and wrote zero; `occInfo` named Inventory's events or streams table across the cells and its `retryCount` was zero, one refused execution per rejected command, and the rerun count remains an observation per record (E-47, F1) - observedUnrelatedStock: [extension] at the native backend tier with `observedMeasurementPins`, eight simultaneous ten-line commands on disjoint stock in one tenant and in eight unrelated tenants each recorded 8 executions, 8 applied outcomes and zero engine reruns; each command read 35 documents and 14454 bytes and wrote 24 documents and 18410 bytes, and the shared generation row was unchanged (E-47, F1) +- observedLatency: [extension] on a quiet ThinkPad laptop running nothing else, at the native backend tier with `observedMeasurementPins` and the backend binary of sha256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae`, the latency cell of `npm run measure:place-order -- --quiet` over 30 accepted `PlaceOrder` commands per size after 5 discarded read a median of 20.3 ms and a 95th percentile of 21.3 ms at 1 line, 35.1 ms and 52.4 ms at 10 lines, and 166.3 ms and 178.3 ms at 100 lines, so every latency target `targetLatency` states is held; these are one local backend's times and say nothing of a hosted deployment (E-47, First experiment, Sc L2-9) +- observedThroughput: [extension] with the same machine and pins, the throughput cell's 8 callers placing ten-line orders on disjoint stock completed 864 applied orders in 10.06 seconds, 85.9 orders a second, with zero engine reruns and no other outcome, beside the local backend's write cap of 4,194,304 bytes a second, so the target of 20 orders a second `targetThroughput` states is held (E-47, First experiment, F13) - observedStockFoldGap: [extension] at the native backend tier with `observedMeasurementPins`, after `ReceiveStock` recorded quantity two, an admin change of current on-hand stock to three with the same stream version and journal tail let a one-line order allocate three, reading 8 documents and 2943 bytes and writing 6 documents and 4271 bytes; matching the tail version does not establish equality of current state with the complete event fold (E-47, D3) - streamBudgets: the stock item stream declares `budgetBytes` of 16 KiB and the order stream keeps the default 256 KiB, as `spec:application.orders-inventory-example` pins, so the allocation call's byte bound, 8 MiB over the planned streams' budgets, admits 512 stock streams and the adapter's count bound of `min(maxStreams, 256)` admits the 100 of the maximum order with room, while the single order stream costs at most 256 KiB; at the default budget on both stream types the byte bound would admit only 32 stock streams and the maximum run would be rejected `operationTooLarge`, which is why the budget is pinned here and the run at the maximum measures it (E-2, E-22, E-46, F13, Sc L2-3) - productDecisions: the maximum order size and the speed targets of `targetLatency`, `targetThroughput` and `targetContention` are set before the benchmark and recorded with the run; the speed targets are set for the pinned local backend on the machine of the run and say nothing of a hosted deployment; a target that is missed is reported as not held with its number, and no target is changed after the run (First experiment, OQ3) diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md index 15ce284..aad8f87 100644 --- a/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.sdp.md @@ -10,7 +10,7 @@ relations: F19 · Status: probed · Doc status: not in the doc's ledger · Decisions: D2, D7, D10. -A nested or component call runs through an asynchronous system call of the pinned package, and the pinned backend does not charge the time a mutation spends waiting on its calls to the one second of execution F13 lists. It charges it to a budget of system operations, whose refusal has its own error. Probe 9 and Probe 11 ran on the native backend, release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0: a mutation that made a thousand empty calls committed after several seconds, and one that made more was refused about fifteen seconds after it began, in every run; a loop of computation was refused at one second with the execution error. The pinned package declares no duration and no metric for the budget, so the count of calls one mutation may make is not a number, and it falls as each call does more. The runs were taken while other work shared the machine, and their times are recorded and not asserted. +A nested or component call runs through an asynchronous system call of the pinned package, and the pinned backend does not charge the time a mutation spends waiting on its calls to the one second of execution F13 lists. It charges it to a budget of system operations, whose refusal has its own error. Probe 9 and Probe 11 ran on the native backend, release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0: a mutation that made a thousand empty calls committed after one to three seconds, and one that made more was refused about fifteen seconds after it began, in every run; a loop of computation was refused at one second with the execution error. The pinned package declares no duration and no metric for the budget, so the count of calls one mutation may make is not a number, and it falls as each call does more. The times are recorded and not asserted; the ones `nativeMeasurements` states were taken on a quiet machine. ## Intent @@ -30,7 +30,7 @@ A nested or component call runs through an asynchronous system call of the pinne ## Design - systemBoundary: A mutation that repeatedly calls an empty nested or component mutation fails with `Your request timed out performing too many system operations.` while its completion record reports zero documents and bytes read and written. A loop of computation fails with `Function execution timed out (maximum duration: 1s)`. Both errors are bound, while elapsed times and requested call counts are recorded (F19, Probe 9, Probe 11) -- nativeMeasurements: With other work on the machine, a thousand empty nested calls committed in 3.8 to 9.4 seconds and a thousand empty component calls in 5.0 to 7.1 seconds of top-level completion time, and every refused request ended between 15.2 and 15.7 seconds; these are one local backend's times under load, recorded in each run's evidence and asserted by no example (F19, Probe 9, Probe 11) -- componentCost: Probe 11 compares transaction metrics before and after twenty component calls and records the top-level completion usage and time. Empty calls read and write zero documents. Each control call inserts one document and reads it back, costing one document read and one written; a call took several milliseconds under load, and the times are recorded and not asserted (F19, Probe 11) +- nativeMeasurements: On a quiet ThinkPad laptop running nothing else, on the release `precompiled-2026-09-28-5c7cb5b` of sha256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae` with `convex` 1.46.0 and Node v24.19.0 on Linux 7.2.7-200.fc44.x86_64 x64, a thousand empty nested calls committed in 1.83 seconds of top-level completion time and eight thousand in 14.71, a thousand empty component calls in 3.00 seconds and four thousand in 12.18, the refused requests of sixteen thousand nested and eight thousand component calls ended at 15.88 and 15.62 seconds, and the loop of computation was refused after 1.00 second of wall time; the timing command's component reading on the fixture composition, 100 reads per sample over 9 samples per path, has medians of 7.2 ms through a helper, 208.6 ms through nested calls and 324.6 ms through component calls, 3.17 ms per component call over a helper; these are one local backend's times, recorded in each run's evidence and asserted by no example (F19, Probe 9, Probe 11, F4) +- componentCost: Probe 11 compares transaction metrics before and after twenty component calls and records the top-level completion usage and time. Empty calls read and write zero documents. Each control call inserts one document and reads it back, costing one document read and one written; on the quiet machine of `nativeMeasurements` an empty call took 3.50 ms and a control call 3.51 ms, and the times are recorded and not asserted (F19, Probe 11) - countBoundary: The probe increases the requested count until the backend refuses it, recording every successful count and the first failed request. The refusal can end execution before JavaScript catches the error, so the failed request's exact completed-call count is not observable by the return value. No count is bound as a platform limit (F19, Probe 11) - factCoverage: Probe 11 verifies this time-budget fact. It records component-call costs that inform the concern of F4, but does not verify F4's comparison against a plain helper because it has no helper baseline (F19, F4, Probe 11) From ab863ddc842acbee399e592f13e6b5683bc82114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:21:54 +0200 Subject: [PATCH 072/155] design: the observed PlaceOrder documents name no gate document and what the gate read adds Co-Authored-By: Claude Opus 5.5 --- design/specs/application/first-experiment.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 1cc5a91..d24ee99 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -117,7 +117,7 @@ The experiment is an application, a test suite and a report. The application is - limitOrderLines: 100 lines per `PlaceOrder`, the largest order the example supports and the bound its declaration carries, chosen under the 1 second timeout, the F13 ceilings and the adapter's stream and byte bounds; it is the example's promise and no platform limit, and the experiment reports documents read and written per line (OQ3, F13) - limitStockItemIdBytes: [extension] 64 bytes of UTF-8 for the stock item ID of each order line of `PlaceOrder` and each item of `ReceiveStock`, the bound both declarations carry and check in their refinements, so a longer one is refused `invalidInput` at step 1 of the command pipeline, before any read and before any context is called, as the stock item ID bound of `spec:application.orders-inventory-example` states; it is chosen so that the maximum order's `OrderPlaced` fits the 16,384-byte payload bound of E-12: `getConvexSize` measures the payload at 25 + 100 × (53 + 64) = 11,725 bytes at the bound, where 128 bytes would make 18,125 and the 256 bytes of a stream ID 30,925 (E-46, E-12, OQ3, F13) - limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) -- observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate read, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer (E-47, F13) +- observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate document, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer; the gate read of the pipeline's step 7 runs in every one of these commands, and with no gate document in the table it finds nothing and adds no document read and no byte, so the counts above include it, and no measurement here has a gate document in the table (E-47, F13, E-8) - observedMeasurementPins: [extension] the measurement records name the native backend tier, the local backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124, `convex-test` 0.0.60, Node v24.19.0 and Linux 7.2.7-200.fc44.x86_64 on x64; these observations concern that local environment only (E-47, E-15, Sc L2-9) - observedMeasurementBytes: [extension] at the native backend tier with `observedMeasurementPins`, fixed seven-character stock and order IDs, four grants and one active generation, orders of 1, 10 and 100 distinct stock items received once read 2943, 14454 and 129564 bytes and write 4271, 18410 and 159800 bytes; the corresponding write-index rows are 25, 97 and 817; a ten-line authorized duplicate or changed-input conflict reads 5 documents and 3441 bytes and writes zero, while short stock or a missing journal tail on the last line reads 32 documents and 13106 bytes and writes zero (E-47, F13) - observedContentionCosts: [extension] at the native backend tier with `observedMeasurementPins`, each of the nine cells at 1, 10 and 100 lines and 2, 8 and 32 callers reached one applied outcome and all remaining short-stock rejections with zero engine failures; an execution refused at commit read the healthy path's documents and bytes and wrote zero, and its final short-stock rejection, with the shared item first, read 5 documents and 1595 bytes and wrote zero; `occInfo` named Inventory's events or streams table across the cells and its `retryCount` was zero, one refused execution per rejected command, and the rerun count remains an observation per record (E-47, F1) From 8cc40e8163c71ad870cd6b58749eee4bd0998bfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 01:40:28 +0200 Subject: [PATCH 073/155] design: the close of round 2 of the roadmap, completed The second review pass, its fold-in (J1 to J8) and the sol read of it fill the marked line of the review; the quiet timings of ruling J7 fill the line of the timings, every F2 target held, and the owner queue no longer waits for them. "Next unit" adds the owner's question on a hosted tier as the second subject of the discussion. The measured lines are those of the new head. Co-Authored-By: Claude Opus 5.5 --- design/ROADMAP.md | 4 ++-- design/STATE.md | 17 ++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/design/ROADMAP.md b/design/ROADMAP.md index 05881d8..ac5246b 100644 --- a/design/ROADMAP.md +++ b/design/ROADMAP.md @@ -16,7 +16,7 @@ The slices of `STATE.md` keep their names. S3 is round 1's attribution and round | 1 | b | Attribute the 3N + 5 reads of `PlaceOrder` under controlled changes. Done (#9) | build | convex | an evening | nothing | | 1 | c | Confirm the thirty-three flagged fixes in the Spec text. Done (#6) | design | product | an evening | nothing | | 1 | d | Scout: does the pinned local backend take a snapshot import with replacement. Done, yes (#7) | build | operator, convex | an afternoon | nothing | -| 2 | a | The S3 measurement: six cells, Measurement records, the six targets, Sc L2-9's concurrency half. Done: in every contention cell one command applied and the rest were rejected, the six cost targets held, the timings wait for a quiet run | build | convex | a night | 1b, OD-046, OD-064 | +| 2 | a | The S3 measurement: six cells, Measurement records, the six targets, Sc L2-9's concurrency half. Done: in every contention cell one command applied and the rest were rejected, the six cost targets held, and on a quiet machine every speed target held | build | convex | a night | 1b, OD-046, OD-064 | | 2 | b | S4 on paper: the per-entity generation lifecycle, the gate and the audit record, the operator entry contract. Done, by one designer: installation through the first rebuild, the tenant list, the operator entry contract, S4's open findings 31 to 19 | design | domain, operator | an evening | sitting one | | 2 | c | `CancelOrder` and `release` in the example domain. Done: totals only, six native examples, a cancel writes six documents | mixed | domain | an evening | the owner's answer on allocations | | 2 | d | Probe 6, before the rebuild is built on the component. Done: the component does not drive a context's rows as documented, so the rebuild drives every batch itself | build | convex | an evening to a night | OD-057's checked lean applied | @@ -69,7 +69,7 @@ Done on 2026-10-02 and 2026-10-03, as nine lanes merged into `feature/round-2`, The owner discusses the next session's work before it is prepared. In this order: 1. **3b, the online rebuild of the order summary**, Sc L2-5 at the native tier. It is the next unit. What it needs is in `design-s4-paper.md` in the round 2 folder: package 3b of section 3 and the rulings of section 7. 3a is merged, so `assertOperator`, `nextTenant` and `gateAllows` exist. The files: `src/read-model/projection.ts` (`projections`, `projectionOf`, the three roles of a written generation, the backfill mode), `src/read-model/tables.ts` (the slimmer `generations` and `generationProgress`, with its field `pass`), `src/read-model/generations.ts` (the roles, and `activateFirstGeneration` removed), a new `src/read-model/rebuild.ts` (one helper per entry, `RebuildConfig`, `batchSizeFor`, `resumeChain`), the order summary's `projections`, the composition's registered rebuild functions and the fixture's read models the same. Every setup that ran the first activation installs through `startGeneration`, its batches and `switchGeneration`, and the first-activation example is renamed `first-rebuild-installs-read-model` with its test in one commit. From section 7: `fillTenants` and its batches with the refusal of `startGeneration` on an empty tenant list beside a grant, `resumeGeneration` with no batch size, the backfill mode's rule for a row that projects to `null`, and the message that sends a started purge to `resumeGeneration`. The proof obligations are at the pure, `convex-test` and native tiers, the native one the L2-5 example: 350 orders in two tenants, batch size 1, live commands in both tenants, an interrupt, a resume, a switch and a rollback. It does not build the paused path, the aggregate rebuild, `@convex-dev/migrations` in any form, an automatic stall check, a purge that starts by itself or the restore's `finishRestoreRun`. -2. **3d, sitting two.** The language rows OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added (snapshot, tenant fill, the measurement's words); OD-008 with OD-006; OD-138; OD-041, OD-042 and OD-114, the last with Probe 7's facts; OD-044 with F18 against D2; OD-063 with its local numbers; OD-048, the licence; and the two speed targets of F2 once their timings are taken on a quiet machine. +2. **3d, sitting two.** The language rows OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added (snapshot, tenant fill, the measurement's words); OD-008 with OD-006; OD-138; OD-041, OD-042 and OD-114, the last with Probe 7's facts; OD-044 with F18 against D2; OD-063 with its local numbers; and OD-048, the licence. The F2 comparison is done: every target held on the quiet ThinkPad. 3. **4a, command audit and the diagnostic.** 3a has built `writeAudit` and step 10, so what is left is the Spec text on which outcomes emit a record and where a duration comes from, the sink, and Sc ALL-1 bound. 4. **4c, the history view on paper.** 2c is done, so `OrderCancelled` and `AllocationReleased` exist for its fold. 5. **4d, the receipts sweep.** The tenant list and `nextTenant` are built. diff --git a/design/STATE.md b/design/STATE.md index 29d2302..d6bf44e 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -11,7 +11,7 @@ open questions: 69 Specs, 147 questions, 9 blocking; extensions registered: 53 stated readiness: {'defined': 222, 'idea': 1, 'scoped': 9} ledger: 278 findings, {'fixed': 148, 'open': 103, 'owner': 1, 'partially-fixed': 26} open findings by slice: {'L3': 31, 'P': 30, 'S3': 1, 'S4': 19, 'S5': 22} -corpus digest: e1a6ac63b9bc1390 +corpus digest: c4805deb1c987a46 protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` @@ -32,7 +32,7 @@ Round 2 of the roadmap, from the evening of 2026-10-02 to past midnight, on the The main thread ruled the forks of sitting one on the owner's word, as F1 to F12 of `rulings-1-round-2.md` in the unit folder: twelve rows decided (OD-046, OD-064 and OD-123 by one answer, OD-153, OD-068, OD-069, OD-148, OD-025, OD-150, OD-151, OD-152, OD-047) and OD-048, the licence, left the owner's. The register tool refused a session ruling on an `owner` row, the owner chose the cleanest path, and the twelve are recorded as decided by the owner, each ruling ending with both of the owner's sentences; the tool is unchanged (decision 40). OD-007 was decided by its advisor, its checked lean as the ruling. What each lane showed, at the tier named: - Sitting one. An Opus lead wrote the twelve answers into the rows and the forks and applied F1 to F3 at ten Specs: 100 lines is the largest order `PlaceOrder` supports, the bound its declaration carries and no platform limit; the speed targets of F2 are set for the pinned local backend before the benchmark; Sc L2-9 is read as the local observation, its concurrency half the contention run. A sol check found the twelve ruling texts and the thirteen fork answers exact and six Spec sentences that still contradicted a ruling, all corrected. Open questions fell from 144 to 138 at that lane. -- The S3 measurement (2a). `gpt-6-astra` at high, 20 minutes, one turn, on the native backend, local: ten cost cells and nine contention cells, each writing a `Measurement` record. The healthy order of 1, 10 and 100 lines read and wrote 8/6, 35/24 and 305/204 documents; a duplicate or a changed-input conflict read 5 and wrote none. In every one of the nine contention cells one command applied, N minus 1 were rejected for short stock and the engine failed none: 3, 15 and 63 executions at N of 2, 8 and 32, each loser refused once at commit and rejected on its rerun; the worst cell, 100 lines at 32, read 9,915 documents and 4,195,493 bytes for one applied order. Eight commands on unrelated stock, in one tenant or eight, applied with no rerun. The six cost targets held, the fourth as C1 counts it. A stock state that admin access set ahead of its events let an order for 3 apply against 2 received: a ledger finding for S5 (C2), no check on the command path. The journal tail read and the four-document grant read stay (C3, C4), with their costs on OD-066 and the new OD-154. The timings were taken only under load, and two readings 6 times apart at one line say the load and not the cost, so no speed target is compared yet. +- The S3 measurement (2a). `gpt-6-astra` at high, 20 minutes, one turn, on the native backend, local: ten cost cells and nine contention cells, each writing a `Measurement` record. The healthy order of 1, 10 and 100 lines read and wrote 8/6, 35/24 and 305/204 documents; a duplicate or a changed-input conflict read 5 and wrote none. In every one of the nine contention cells one command applied, N minus 1 were rejected for short stock and the engine failed none: 3, 15 and 63 executions at N of 2, 8 and 32, each loser refused once at commit and rejected on its rerun; the worst cell, 100 lines at 32, read 9,915 documents and 4,195,493 bytes for one applied order. Eight commands on unrelated stock, in one tenant or eight, applied with no rerun. The six cost targets held, the fourth as C1 counts it. A stock state that admin access set ahead of its events let an order for 3 apply against 2 received: a ledger finding for S5 (C2), no check on the command path. The journal tail read and the four-document grant read stay (C3, C4), with their costs on OD-066 and the new OD-154. The lane's own timings were taken under load, two readings 6 times apart at one line, so the comparison waited for the quiet run below. - The S4 paper (2b). One Fable designer, 18.5 minutes, both subjects, Spec text only: a read model installed through its first rebuild, the projection chosen by the version its generation row records, progress on a `generationProgress` row only batches read, one verify pass that inserts, updates and deletes, a switch of one write, rollback as F4 rules it, `interruptGeneration`, the tenant list as the `tenants` table (E-49), the operator entry contract of F6 (E-48), the gate's scopes and refusal, and `AuditRecordInput` pinned. `design-s4-paper.md` in the unit folder has the decisions and the build cut. A sol check at medium in 8 minutes listed fifteen first findings: an Opus lead applied 11, sent 2 to the main thread and left 2 with a reason, found 3 sol did not, and cut sol's ledger draft of 37 items to 25. S4's open findings went from 31 to 19. The main thread's six rulings after the check are section 7 of the note. - `CancelOrder` and `release` (2c). An Opus builder, then a sol read at medium with 35 mutations. Pure, `convex-test` and six native examples on the production composition: the order's `cancel` and `OrderCancelled`, the stock item's `release` and `AllocationReleased` with totals only (F9), Orders called first, the summary row replaced with `cancelled`, three new rejections. A cancel writes six documents on the native backend, and a refused release writes none. Nothing under `src/` changed. Sol's read led to twelve corrections, each new assertion shown to fail with its rule removed, and found that an `OrderPlaced` of 100 lines with 256-byte stock item IDs is 30,925 bytes, above the 16,384 bytes `append` holds a payload to. - Probe 6 (2d). Astra at high, two turns, 40 minutes, six native examples on the fixture composition: `@convex-dev/migrations` 0.3.6 walks only the tables of the component it is defined in; it drives a context's rows only through a parent internal mutation written by hand, against a contract its README does not document, which the compiled tier needs a cast for; its status and the generation's state are two vocabularies; a batch that reads past 16 MiB fails as a whole and leaves no error on the component's row. So the rebuild drives every batch itself and uses the component for nothing (G2), F17 is probed, and the component is pinned as a dev dependency for the fixture's probe only. @@ -42,9 +42,9 @@ The main thread ruled the forks of sitting one on the owner's word, as F1 to F12 - Polish. An Opus lead and sol at medium, 5 minutes. A native reproduction showed E2's premise: a 100-line order with 256-byte stock item IDs passed step 1, `append` threw a plain error inside Orders, and the caller got a bare technical failure with nothing stored. `PlaceOrder` and `ReceiveStock` now refuse a stock item ID above 64 bytes of UTF-8 at step 1 as `invalidInput`: 100 lines at 64 bytes make a payload of 11,725 bytes, at 128 bytes 18,125. On the native backend an order at the bound applied with 305 documents read and 204 written, and one byte past it was refused with none read. Five mutations ran red. The stale README counts were recomputed. - The integration. An Opus integrator merged eight lane branches into `feature/round-2` on top of sitting one. The conflicts were counts, a few bullets two lanes had both written, and generated files, which one codegen after each merge left unchanged. One test disagreement between two lanes (the measurement's tables left behind against the gate's new tables) was a wrong test. At `e6e1112`, every project in one run with other lanes on the machine: 160 files, 761 tests, 754 passed, 1 failed and 6 skipped; the failure was a literal parent table name in a Probe 6 example that the gate's new table shifted, since fixed. `npm run acceptance` on that record: 31 scenarios passed, none failed or absent, 5 missing (Sc L2-5, L2-6 twice, L2-7, L2-8, which have no verifier yet). - The review. Astra at xhigh, 13 minutes, on a frozen copy of `1b92bd8`: two findings on an ordinary path (the Probe 6 fixture callback writes a wrong status, `deploySnapshotFixture` drops the caller's signal), four surviving mutations, and two places where the S4 text did not determine the behavior (a deployment whose grants predate the tenant list, the batch size of a resumed purge). With 8 mutations and the gate and polish lanes outside its copy it was thin for xhigh, so a second pass reads the head after the fold-in. -- The second pass of the review, astra at xhigh on a frozen copy of `40ea25d`. `[review pass 2]` +- The second pass of the review, the same astra job at xhigh, 16 minutes, on a frozen copy of `40ea25d` with the gate, polish and fold-in in it. The whole native tier passed, 144 tests and 6 skipped. Of 34 mutations 29 were killed, and every fold-in test failed under its mutation. It found one defect on a healthy path: `getGateAudit` let a caller's `endCursor` widen a page past the cap of 100, which only admin access reaches. Beside it: a tenant ID with no bound in `insertGrant`, a sentence that did not say whether `fillTenants` reads the restore door, three measurement assertions that passed under mutation, and Probe 7's restore comparisons leaving out the four new tables. A second fold-in closed all of it as rulings J1 to J8, each new test shown to fail under its mutation (`convex-test` for the page cap and the tenant ID bound, a pure test for a decomposed stock item ID, native for the measurement and Probe 7). A sol read of that fold-in at medium, 7 minutes, found nothing on a healthy or ordinary failure path and recomputed every number with no difference, so the round stopped on tactical decision 19. - The fold-in. An Opus agent with no GPT job, because the rulings named every change: the caller's signal reaches the deploy child (pure test), three `CancelOrder` tests at `convex-test` over two stock items, a second tenant and two orders, each shown to fail under its mutation, and the Spec text of `fillTenants`, the refusal of `startGeneration` on an empty tenant list beside a grant, and a resumed pass keeping its batch size. At `40ea25d`: typecheck, lint and format pass, `npm test` 59 files and 615 tests, the six native files that reach `deployTemporary` pass, `check.py` OK. -- The timings on a quiet machine. `[timings]` +- The timings on the quiet ThinkPad, at `8421d6b` with no other job running (ruling J7; the logs and pins are `timings-place-order-quiet.log` and `timings-f19-quiet.log` in the unit folder), on the native backend, local. `PlaceOrder`'s latency, median and 95th percentile: 20.3 and 21.3 ms at 1 line, 35.1 and 52.4 ms at 10, 166.3 and 178.3 ms at 100. Throughput: 85.9 ten-line orders a second from 8 callers over 10 seconds, 864 applied, no rerun, beside the local write cap of 4 MiB a second. Every F2 target held, the contention target included. F19's probe, 100 reads a sample: a median of 7.2 ms through a helper, 208.6 nested and 324.6 through a component, 3.17 ms per component call over a helper call. The Specs carry them: `observedLatency` and `observedThroughput` in the first-experiment Spec, F19's `nativeMeasurements`. The branches. Each lane worked in `~/dev-libar/wt/` on a branch of its own, from `main` at `592f1dc` for the first seven and from `feature/round-2` or `unit/s4-paper` for gate and polish, and the integrator merged each into `feature/round-2` in `~/dev-libar/wt/round-2`. The branch is pushed. The owner's words: "we will need a feature branch", then "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue". So no pull request goes to `main` until the owner has reviewed `feature/round-2`. @@ -175,7 +175,9 @@ Twenty-six more findings are `partially-fixed`, each waiting for its slice. Remo `ROADMAP.md` is the order, ruled as cut by decision 36, and rounds 1 and 2 are done. The next subject is the online rebuild of the order summary, roadmap 3b, with Sc L2-5 at the native tier: package 3b of `design-s4-paper.md` in the unit folder names its files, its order, its proof obligations and what it must not build, and section 7 adds the rulings after the check, `fillTenants` and the resumed batch size among them. The owner said on 2026-10-03: "We can discuss details before preparation of work for the next session." So the main thread lays the next session's work out with the owner before any brief is written or any lane starts. -`slice` S3 is round 1's attribution and round 2's measurement; its last part is the timings on a quiet machine. S4 is round 2's paper and gate, and the builds of rounds 3 and 4. S5 is round 5. The paper decisions S4's rebuild needs are taken; those of the history view and of S5 are listed below. +The second subject of that discussion. The owner asked on 2026-10-03 whether to start de-risking the implementation with tests against a real Convex backend and a minimal setup based on `convex-event-sourcing`, and said design work for tooling could be commissioned separately. The main thread answered that the native tier already runs against the real backend, and that the risk no tier covers is a hosted deployment: F12 and F16 stayed documented and OD-036, Probe 3's quota half, is unrun. It recommended a read-only astra scout over the earlier platform's tooling under the two mining rules (a fact with its date and versions, none of its designs or habits), then a small `build` unit for a hosted tier gated by a deploy key and run on a schedule, with the deployment arrangement designed first. + +`slice` S3 is round 1's attribution and round 2's measurement, the quiet timings included. S4 is round 2's paper and gate, and the builds of rounds 3 and 4. S5 is round 5. The paper decisions S4's rebuild needs are taken; those of the history view and of S5 are listed below. How S2 was run, for the next slice. Tactical decisions 25 to 27 hold the rules. Two Fable designers with disjoint subjects took about twenty minutes. One workflow run of thirteen Opus agents then wrote the Specs, built four packages in sequence, bound the scenarios in four worktrees and integrated, in about an hour and a half. The main thread stopped and resumed that run once, at a seam between two packages, to add corrections: finished agents came back from the cache. The scripts, the rulings and the reports are in the S2 folder the project notes name. @@ -273,11 +275,10 @@ Sitting two waits for the owner. Its forks are drafted the same way as sitting o - `OD-048`, the licence, the one fork of sitting one left the owner's. It blocks nothing before round 5c. - `OD-044`, D2's premise, with F18 against it: on the pinned backend one transaction's rows share one commit timestamp in the parent and in components and later transactions get larger ones, while D2 and `spec:context.journal` say no order token for the whole deployment exists. Nothing uses the internal `getSnapshotTs()`. -- `OD-063`, OQ1, a component per context, with its local numbers on the lean: a fixture component call cost 3.56 and 5.76 ms over a helper call in two readings under load, and Probe 11 read 7.6 to 15.5 ms per empty component call, all at the native backend tier on the local backend. The quiet reading is still to come. +- `OD-063`, OQ1, a component per context, with its local numbers on the lean: a fixture component call cost 3.56 and 5.76 ms over a helper call in two readings under load, and Probe 11 read 7.6 to 15.5 ms per empty component call, all at the native backend tier on the local backend; on the quiet ThinkPad, 3.17 ms per component call over a helper call. - `OD-114`, D13's do-nothing check, with Probe 7's facts: a restore in place keeps schedules the restored data no longer explains, a fresh restore keeps none, and a scan of `_scheduled_functions` stops at about 4,088 outstanding rows on the read limit. - The language rows, OD-009, OD-022, OD-029, OD-139 and OD-140, with the words the round added: "snapshot" for the exported archive, which `CONTEXT.md` lists to avoid for state; "tenant fill" for `fillTenants`, `fillTenantsBatch` and `tenantFill`; the measurement's words (measurement record, cell, execution against attempt, rerun, contention, controlled change, local backend against hosted deployment); and the words of `ROADMAP.md`'s list. - OD-008 with OD-006, OD-138, and the S5 rows OD-041 and OD-042, as round 3's sitting two cuts them. -- The two speed targets of F2 that are not yet compared, latency and throughput, because their timings wait for a quiet run (`[timings]` in "Where the work stands"). F2 says a target that is missed is reported as not held with its number and none is changed after the run. `delegated` rows that wait for the next sort of their lens, each with the round's result on its lean: OD-057 (Probe 6 and the rebuild's own batch driver), OD-066 (3N + 5 with every term attributed; the journal tail read is 10 documents and 6,010 bytes, 29 percent of the documents read on ten lines), OD-120 (Probe 7's states) and OD-154 (3 documents and 918 bytes for the grant read) for the convex lens, and OD-081 (Probe 7's restore answers) for the operator lens. The session may not decide a `delegated` row (`decisions.py`: "session may decide only tactical rows"). @@ -357,8 +358,6 @@ A session confirms or drops each. None is a finding yet. - The Probe 6 context callback at `fixture/convex/migrations.ts:198-203` writes a `draft` summary beside the source's version whatever the source's status. Fixture probe code that nothing rests on; fixed only if that fixture is reused. - A partial tenant list passes `startGeneration`'s refusal: a list that holds a tenant whose first grant came after the table, and misses an older one, gets through. The rebuild Spec states it as an open question and tells the operator to run `fillTenants` first. `limitTenantFillBatch` is 8 because a grant row is bounded only by the 1 MiB document limit. - An event payload above the 16,384 bytes `append` holds it to reaches the caller as a bare technical failure: `append` throws a plain error inside the context, and nothing is stored. The example's stock item ID bound now refuses the case that reached it first; any command whose payload can outgrow the bound meets the same failure. - - The timings not yet taken on a quiet machine. `PlaceOrder`'s latency and throughput: `npm run measure:place-order -- --quiet`, from `record-s3-measurement.md`. F19's nested and component calls: the command in `record-fact-rows.md`, whose times go into F19's `nativeMeasurements`. - - `first-experiment.sdp.md` `observedPlaceOrderDocuments` says "no gate read". Every command now reads the gate, and with no gate document the read counts no document, so the six native files that count reads pass unchanged: the bullet should say "no gate document". - From the gate: a duplicate is answered from its receipt in a closed scope and under the restore door, as the Spec now says; the plain error for an audited declaration with no subject has a built message and no pinned one; `baseline-operations.broken-audit-aborts` stays unbound until roadmap 4a builds an audited command. - From `CancelOrder`: an invariant that exempts a cancelled order from its time and total checks survives every test; the bounds of the two context operations, Orders' 32 streams and Inventory's 100, have no direct `convex-test` case; `CancelOrder` declares no `bounds`, so a long order ID is refused by the adapter at 256 bytes after authorization and the receipt read. - From the S4 paper: E-48 and E-49 are numbers of package D's range on a package C Spec, because C's range is full; the journal's baseline driver is not written as an operator entry, and the baseline writer changes sources without maintaining read models; the receipts sweep loops with `nextTenant`, which `receipt-table.sdp.md` does not say yet; a scheduled mutation that loses a race is assumed to rerun until it commits, and no pinned package shows a bound. From 8707403c62ef6b289374ed9808e88bccf84afbf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 03:06:42 +0200 Subject: [PATCH 074/155] design: the close of the tooling unit, design only The scout's facts as a proven patch not applied, the 3b layout, one Fable advisor's three memos checked by sol, the hosted tier designed on paper, and the register's seven rows recorded from the advisor's patch. No code and no Spec changed. Co-Authored-By: Claude Fable 5.1 --- design/ROADMAP.md | 22 ++++---- design/STATE.md | 99 ++++++++++++++++++++++++++++++---- design/decisions/register.json | 74 ++++++++++++++----------- 3 files changed, 145 insertions(+), 50 deletions(-) diff --git a/design/ROADMAP.md b/design/ROADMAP.md index ac5246b..b489a94 100644 --- a/design/ROADMAP.md +++ b/design/ROADMAP.md @@ -1,6 +1,6 @@ # Roadmap: the next units -Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of round 2 on 2026-10-03. +Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of the tooling unit on 2026-10-03. ## How to read it @@ -24,7 +24,8 @@ The slices of `STATE.md` keep their names. S3 is round 1's attribution and round | 2 | f | Also done in round 2: the stock item ID bound, so the largest order fits the payload bound `append` holds an event to; 64 bytes, a refusal at step 1 | build | domain | under an evening | 2c | | 2 | g | Also done in round 2: the review's fold-in, the caller's signal in the harness, three `CancelOrder` tests, `fillTenants` and the resumed batch size on paper | mixed | operator, domain | under an evening | the review | | 3 | a | The gate, the restore door, and a gate change as audit that fails closed. Done in round 2: 106 proof tests written apart from the fix, all passing at the merge, the fix wrong nowhere | mixed | operator | an evening | 2b | -| 3 | b | The online rebuild of the order summary, Sc L2-5 at the native tier | build | domain | a night | 2b, 2d, 3a | +| 3 | b | The online rebuild of the order summary, Sc L2-5 at the native tier: thirteen functions per composition, laid out in `layout-3b.md` of the tooling unit as three lanes and 33 proof obligations | build | domain | a night | 2b, 2d, 3a, the layout's eight questions ruled, the five register rows recorded | +| 3 | e | The hosted tier, commissioned from `design-hosted-tier.md` of the tooling unit: four hosted examples on one shared development deployment, the local backend primary | build | convex, operator | unknown until the first run | 3b, the deploy key and its secret, `patch-hosted-specs.diff` applied, sitting two's words | | 3 | c | The two harness reads and the fact rows no Spec carries. Done in round 2: F18 to F20 probed, the "no job" assertions read `_scheduled_functions`; the deployed-function read stays deferred | mixed | convex | a session plus an evening | nothing | | 3 | d | Sitting two: the language, the pass sentence, and the S5 rows | decide | product, domain, operator | an evening of forks, one check, one owner sitting | nothing | | 4 | a | A command's audit fails closed and its diagnostic never aborts, Sc ALL-1 | mixed | operator | an evening | 3a | @@ -68,17 +69,20 @@ Done on 2026-10-02 and 2026-10-03, as nine lanes merged into `feature/round-2`, The owner discusses the next session's work before it is prepared. In this order: -1. **3b, the online rebuild of the order summary**, Sc L2-5 at the native tier. It is the next unit. What it needs is in `design-s4-paper.md` in the round 2 folder: package 3b of section 3 and the rulings of section 7. 3a is merged, so `assertOperator`, `nextTenant` and `gateAllows` exist. The files: `src/read-model/projection.ts` (`projections`, `projectionOf`, the three roles of a written generation, the backfill mode), `src/read-model/tables.ts` (the slimmer `generations` and `generationProgress`, with its field `pass`), `src/read-model/generations.ts` (the roles, and `activateFirstGeneration` removed), a new `src/read-model/rebuild.ts` (one helper per entry, `RebuildConfig`, `batchSizeFor`, `resumeChain`), the order summary's `projections`, the composition's registered rebuild functions and the fixture's read models the same. Every setup that ran the first activation installs through `startGeneration`, its batches and `switchGeneration`, and the first-activation example is renamed `first-rebuild-installs-read-model` with its test in one commit. From section 7: `fillTenants` and its batches with the refusal of `startGeneration` on an empty tenant list beside a grant, `resumeGeneration` with no batch size, the backfill mode's rule for a row that projects to `null`, and the message that sends a started purge to `resumeGeneration`. The proof obligations are at the pure, `convex-test` and native tiers, the native one the L2-5 example: 350 orders in two tenants, batch size 1, live commands in both tenants, an interrupt, a resume, a switch and a rollback. It does not build the paused path, the aggregate rebuild, `@convex-dev/migrations` in any form, an automatic stall check, a purge that starts by itself or the restore's `finishRestoreRun`. -2. **3d, sitting two.** The language rows OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added (snapshot, tenant fill, the measurement's words); OD-008 with OD-006; OD-138; OD-041, OD-042 and OD-114, the last with Probe 7's facts; OD-044 with F18 against D2; OD-063 with its local numbers; and OD-048, the licence. The F2 comparison is done: every target held on the quiet ThinkPad. -3. **4a, command audit and the diagnostic.** 3a has built `writeAudit` and step 10, so what is left is the Spec text on which outcomes emit a record and where a duration comes from, the sink, and Sc ALL-1 bound. -4. **4c, the history view on paper.** 2c is done, so `OrderCancelled` and `AllocationReleased` exist for its fold. -5. **4d, the receipts sweep.** The tenant list and `nextTenant` are built. +1. **3b, the online rebuild of the order summary**, Sc L2-5 at the native tier. It is the next unit, laid out in `layout-3b.md` of the tooling unit. The layout counts thirteen registered functions per composition and 33 proof obligations (1 compiled, 4 pure, 23 `convex-test`, 5 native). It plans three lanes at once (fix, proof, native), then an integrator, a review and a fold-in. Rulings 2 of that unit rules its eight questions. Its Spec source is `design-s4-paper.md` in the round 2 folder: package 3b of section 3 and the rulings of section 7. 3a is merged, so `assertOperator`, `nextTenant` and `gateAllows` exist. The files: `src/read-model/projection.ts` (`projections`, `projectionOf`, the three roles of a written generation, the backfill mode), `src/read-model/tables.ts` (the slimmer `generations` and `generationProgress`, with its field `pass`), `src/read-model/generations.ts` (the roles, and `activateFirstGeneration` removed), a new `src/read-model/rebuild.ts` (one helper per entry, `RebuildConfig`, `batchSizeFor`, `resumeChain`), the order summary's `projections`, the composition's registered rebuild functions and the fixture's read models the same. Every setup that ran the first activation installs through `startGeneration`, its batches and `switchGeneration`, and the first-activation example is renamed `first-rebuild-installs-read-model` with its test in one commit. From section 7: `fillTenants` and its batches with the refusal of `startGeneration` on an empty tenant list beside a grant, `resumeGeneration` with no batch size, the backfill mode's rule for a row that projects to `null`, and the message that sends a started purge to `resumeGeneration`. The proof obligations are at the pure, `convex-test` and native tiers, the native one the L2-5 example: 350 orders in two tenants, batch size 1, live commands in both tenants, an interrupt, a resume, a switch and a rollback. It does not build the paused path, the aggregate rebuild, `@convex-dev/migrations` in any form, an automatic stall check, a purge that starts by itself or the restore's `finishRestoreRun`. +2. **The hosted tier**, a build unit commissioned from `design-hosted-tier.md` of the tooling unit, after 3b. It needs four things first: the deploy key `hosted-ci` and its GitHub secret, both the owner's by hand; the environment `hosted` and its three variables; `patch-hosted-specs.diff` applied; and sitting two's ruling on the words the patch writes. It shows the four hosted examples run by the workflow on `main`: F16's retention, F12's in-place import, the production composition's acceptance and Probe 3's function calls. The local backend stays primary. +3. **3d, sitting two.** The language rows OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added (snapshot, tenant fill, the measurement's words) and those of the tooling unit (R5's four, the hosted patch's, 3b's, the operator term); OD-008 with OD-006; OD-138; OD-041, OD-042 and OD-114, the last with Probe 7's facts; OD-044 with F18 against D2; OD-063 with its local numbers; and OD-048, the licence. The F2 comparison is done: every target held on the quiet ThinkPad. +4. **4a, command audit and the diagnostic.** 3a has built `writeAudit` and step 10, so what is left is the Spec text on which outcomes emit a record and where a duration comes from, the sink, and Sc ALL-1 bound. +5. **4c, the history view on paper.** 2c is done, so `OrderCancelled` and `AllocationReleased` exist for its fold. +6. **4d, the receipts sweep.** The tenant list and `nextTenant` are built. ## Round 3 **3a. The gate and the door.** The gates table, `gateAllows`, `assertWritable` at step 7, close, resume and get for the `all` and tenant scopes, the read of `MAINTENANCE_MODE` before the gate document, and the audit table with `writeAudit` inside every close, resume and abort. At `convex-test`: a command in a closed tenant is refused and nothing is stored, and a close whose audit insert fails leaves the gate open. On the native backend: the environment switch refuses every write, and two writers race a close. -**3b. The online rebuild.** The rebuild's eight functions and the progress row, with the version-selected projection of 2b. A version 2 of the order summary built as generation 2 while `PlaceOrder` runs, a fence bump as the interrupt, verify, switch, intervening writes, rollback, verify again, switch back. Sc L2-5 on the native backend with a record. +**3b. The online rebuild.** The rebuild's thirteen functions per composition (eight operator entries, four batches and `getGenerations`, as `layout-3b.md` counts them) and the progress row, with the version-selected projection of 2b. A version 2 of the order summary built as generation 2 while `PlaceOrder` runs, a fence bump as the interrupt, verify, switch, intervening writes, rollback, verify again, switch back. Sc L2-5 on the native backend with a record. + +**The hosted tier.** Built from `design-hosted-tier.md` section 1, decision 7, in its order: redaction first, then the record shape and the acceptance filter, `hostedTarget`, the selection kinds, the run setup, the local example of the undeclared table, the copy's modules, the four examples and the workflow. It is built when one record carries `target: "hosted deployment"` with no form of the key, the acceptance example passed, and a CI run on `main` says so. Previews, a second deployment, deletion and cost in money are out. The project and the deployment `dev/hosted` exist since 2026-10-03. **3c. The harness reads and the fact rows.** A read of `_scheduled_functions` through admin access, so that the "no job" assertions of Sc L2-2 and L2-3 close on the table and not on a window; three fact Specs written from the pinned package source and a probe on the pinned release, and not from the leads that point at them: the commit timestamp as OD-044's lean says, the time budget nested calls draw on, and the late byte check of `convex-helpers`; the scheduled-argument size discrepancy recorded on F13; one probe counting component calls per mutation on the pinned release. The native read of the deployed function list is deferred: OD-021's lean takes the committed generated api as the pinned list, and the main thread judges that compiled check sufficient until a scenario needs more. @@ -105,7 +109,7 @@ The owner discusses the next session's work before it is prepared. In this order ## Deferred, with the trigger - Layers 3 to 6: at their trigger, after Probe 7, as the doc and tactical decision 3 say. OD-050 to OD-054 and OD-056 stay waiting until an activation record names a consumer. -- Probe 3's quota half: once on a hosted deployment after 2a, OD-036. +- Probe 3's quota half, OD-036: no longer deferred. It is the fourth example of the hosted tier, the unit after 3b. - The aggregate form: at a view that needs it, OD-075. The history view is no longer deferred: ruling F10 keeps it in the first experiment, 4c writes its rules and 5d builds it. - Receipt tombstones, the derived mapping and `streamParts`, the import command: when a command needs them, OD-100, OD-113, OD-072. - OD-092's `transactionLimits` probe: before any budget or `maxStreams` is raised; nothing in rounds 1 to 5 raises one. diff --git a/design/STATE.md b/design/STATE.md index d6bf44e..296f12f 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -1,6 +1,6 @@ # State of the design corpus -Written on 2026-10-03 at the close of round 2 of the roadmap, which followed round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. +Written on 2026-10-03 at the close of the tooling unit, which followed round 2 of the roadmap, round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. ## Measured at close @@ -15,7 +15,16 @@ corpus digest: c4805deb1c987a46 protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` -**Round 2 is on `feature/round-2`, not on `main`.** `main` is at `592f1dc`, the close of round 1. The lanes of round 2 branched from `main` or from `feature/round-2` and were merged into `feature/round-2`, which is pushed; CI passed on its pushes at `1b92bd8` and `a5d339d`. No pull request to `main` is opened until the owner has reviewed the branch: tactical decision 42 has the owner's words. The next round's lanes branch from `feature/round-2` and merge back into it. +`python3 design/tools/decisions.py`, after the register patch of this close: + +``` +total: 154 +status: unsorted=0, sorted=100, waiting=22, decided=17, folded=15 +class: tactical=3, delegated=86, owner=50, null=15 +advisor: convex=36, domain=45, operator=34, product=39 +``` + +**Round 2 and the tooling unit are on `feature/round-2`, not on `main`.** `main` is at `592f1dc`, the close of round 1. The lanes of round 2 branched from `main` or from `feature/round-2` and were merged into `feature/round-2`, which is pushed; CI passed on its pushes at `1b92bd8` and `a5d339d`. The tooling unit changed no code and no Spec on the branch: its close commits the register, this file and the roadmap. No pull request to `main` is opened until the owner has reviewed the branch. The owner's words: "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue" and "We can discuss details before preparation of work for the next session." Tactical decision 42 holds them. The next lanes branch from `feature/round-2` and merge back into it. The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Each unit works on its own branch, named for the unit, so that `main` stays green. The owner gave the session a mandate on 2026-10-01 to merge at this early stage, and said the same day that a pull request per unit was not a good idea while the design is iterated quickly. So from slice S1 on there is no pull request: CI runs on every push, and the session fast-forwards `main` once CI is green on the last commit. Slice S0 was the one unit merged through a pull request. That held until the merge of 2026-10-02. @@ -25,7 +34,16 @@ From this merge on, a unit reached `main` through a pull request, tactical decis ## Where the work stands -Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, round 1 of that roadmap is on `main`, and round 2 is on `feature/round-2`. +Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, round 1 of that roadmap is on `main`, round 2 is on `feature/round-2`, and a design unit for the tooling followed on the same branch. + +The tooling unit, `application-platform-tooling-1`, on 2026-10-03 from about 01:50 to 03:00, on the ThinkPad, from `feature/round-2` at `8cc40e8`. The owner's words at the opening: "we will document and start designign the backed tooling updates, not do any changes in the next session. but record data for sure." So no code and no Spec changed on the branch. Each proposed Spec change is a patch in the unit folder, proven on a scratch copy and not applied, and the close commits the register, this file and the roadmap. No native backend started and no hosted deployment ran anything. The main thread wrote two rulings pages, `rulings-1-tooling-1.md` and `rulings-2-on-the-advisor.md`, and ruled on the leads' answers. What each lane showed, at the tier named: + +- Data. `gpt-6.1-sol` at medium, two turns, 9.5 minutes, sorted the eleven facts of round 2's tooling scout by the brief's rule, (a) a fact Spec bullet, (b) a new fact Spec, (c) a native harness Spec bullet, (d) a lead. Three were already carried, four went to the hosted lane, and four are (c): `functionLogOverflow`, `sourceScheduledParallelism`, `sourceHttpMutationFailure` and `sourceBackendOrigin`, Design bullets of the native harness Spec marked `[extension]`, with Probes 12 to 15 proposed as six example Specs. The reproduction of `IsolateNotClean` is (d), lead 35. Every claim rests on source and ran nowhere: the backend at `5c7cb5b`, `convex` 1.46.0 under `node_modules`, docs.convex.dev, this repository's harness. The lead checked the scout's upstream line numbers and corrected five: `knobs.rs:80-82` for the function log's 1,000 entries, `:545-549` for eight scheduled jobs, `:1080-1083` for 4 MiB a second of writes, `isolate.rs:326-335` and `local_backend/src/lib.rs:192-197` with `:208-210`. It sent sol one list of two items, Probe 13's status case to the pure tier and the `convex dev --local` example dropped. The patch, 10 files, proved on a scratch copy: 238 Specs, 373 nodes and 1,241 edges, `check.py` OK, validate 0 errors. It is not applied. Leads 30 to 38 are the lane's. +- 3b's layout. An Opus lead, reading only, wrote `layout-3b.md` from the S4 paper's package 3b and section 7. It counts thirteen registered functions per composition (eight operator entries, four batches and `getGenerations`), where the paper said eleven and the roadmap eight, both written before `fillTenants`. It lists 33 proof obligations: 1 compiled, 4 pure, 23 `convex-test` and 5 native. It plans three lanes at once (the fix by sol; the proof and the native tests by astra), then an integrator, a review and a fold-in, about three to three and a half hours. Its estimate for the native L2-5 run, not a measurement, is about 750 to 780 chained batches: 20 to 50 s on a quiet machine, and 55 to 130 s at round 2's load factor of 2.6, past the native `testTimeout` of 120 s. Sol checked it, reading only, in 4.9 minutes on a copy with one planted citation. The plant was caught, 107 citations held and 5 did not, and the lead fixed each, among them the setup count (39 native and 6 `convex-test` test files reach the first activation) and the estimate's arithmetic. Eight questions went to the owner. +- The one advisor. The owner, after the layout: "please spawn fable advisor to recommend decisions. let's keep it less noisy and use one advisor. quality, simple design, maitanable platform are guiding principles." One Fable advisor of the domain lens took three tasks, each in about five to six minutes by its files' times. The first two were the eight 3b questions (`memo-advise-3b.md`) and the hosted tier's eight decisions (`memo-advise-hosted.md`). The owner answered: "I agree with all recommendations. We have unused Convex Professional account. Let me know how to login." The third was the five register rows the check of the 3b memo sent back (`patch-register-3b.json` with its memo). Sol checked each at medium, reading only, on a copy with one planted citation, and caught every plant. On the 3b memo, 69 citations held and 11 more did not; 29 numbers and quotes held and 5 did not. Four of the memo's reasons were wrong: `documentTitle` resting on a `!` that TypeScript erases, "no tier can show" context calls where a pure test can, two ledger rows called `partially-fixed` that are `open`, and a factory that would halve the surface. Rulings 2 corrects each recommendation they touched. On the hosted memo there were 10 faults, two above the line: the usage fence and serial runs. On the register patch, 55 held and 11 did not, among them OD-075's history form, which changed its `do`, and OD-057's stale `checkedBy`. The advisor rewrote 17 lean fields, and the recheck, 2.7 minutes, found 57 held and 2 did not: OD-149's `ifWrong` and the memo's answer on whether 3b may start. +- Hosted. `gpt-6-astra` at high, 7.9 minutes, one turn, gathered the facts of a hosted deployment from `convex` 1.46.0, the backend at `5c7cb5b` and docs pages read on 2026-10-03. The lead checked all 147 citations against their sources and found 5 that misattributed a quote or a range, none a wrong fact. It added a fourth arrangement beside the scout's three, a fresh development deployment per run, which the deployments page describes. It also added GitHub's rule that a schedule runs from the default branch. The lead's own design draft and patch, written before the main thread chose to spend Fable on the decisions, are kept as `draft-hosted-opus*`: input, not a decision. Sol caught their plant and three high faults. A second Opus lead turned the advisor's memo into `design-hosted-tier.md` and `patch-hosted-specs.diff`, with the owner's fact that the plan is Professional. The design has one shared development deployment in a project and team of their own, one development deploy key scoped to fourteen actions, and one GitHub secret in an environment `hosted` that admits the default branch. Runs are serial. Four hosted examples run in one driver file (F16's retention, F12's in-place import, the production composition's acceptance, Probe 3's function calls), on a cron twice a week. Cost is in the plan's units: Probe 3's fence is 2,060 calls a run, about 18,500 a month against 25M included, and the rest is unmeasured. The 12-file patch changes five Specs, adds five example Specs, and touches the README and the register, where it adds the row OD-155. It proved on a scratch copy (237 Specs, `check.py` OK, validate 0 errors) and is not applied. Sol's check of the design caught its plant and found 14 faults; the lead fixed eleven and sent three forks, ruled in rulings 2. The usage fence stays labelled judgment, and a miss leaves OD-036 open. The retention example passes by planting and recording, which narrows E-16's fail rule. The patch's new words wait for sitting two. Every hosted statement is from source or docs. +- The setup. The owner asked: "This setup cannot be documented and executed by an agent? With gh cli, convex cli, and other tooling?", then "1. and if someting is an issue, i will jump in." An Opus agent ran design section 0 with CLI 1.46.0 at about 02:56. Two things now exist: the project `libar-platform-hosted` in the owner's one team, and its development deployment `dev/hosted`, named `wandering-mule-403`, at `https://wandering-mule-403.eu-west-1.convex.cloud`. `--expiration none` was accepted. The printed URL carries the region, so `HOSTED_DEPLOYMENT_URL` takes the printed form and not the one the design templates. The GitHub environment `hosted`, its three variables, the deploy key and the secret do not exist; they are left to the owner. The agent's own permission check refused the first `gh api` write before it ran, and the brief said to stop at the first refusal. The key was the owner's to make by hand in any case: the main thread's harness refuses to write a secret store even through an agent, a "Secret-Store Writes" denial of 2026-10-03 (decision 47). The CLI left no file. Design step 9, one deployment and no production deployment, stays a dashboard check, because CLI 1.46.0 has no `deployment list`. +- What the owner decided, as rulings 2 records it. 3b's eight questions follow the advisor, with the sharpenings and the corrections after sol's check. The `rebuildFunctions(config)` factory is not tried, because `rebuild.sdp.md:112` keeps thirteen registrations. The setup change covers the eight helpers and the test files that call `activate`. The operator term is a language row for sitting two. The hosted tier takes all eight recommendations on Professional. The setup is an agent's and the key is the owner's. The main thread ruled lane data's forks: `IsolateNotClean` stays a lead, Probes 12 to 15 keep the probe plan's numbering, and lead 33 stands. The register's seven rows are recorded at this close by the tool from the advisor's patch: OD-010 and OD-014 are `decided` by the domain advisor from their checked leans. OD-057, OD-071, OD-075 and OD-080 are `sorted` with new leans that sol's recheck held, which carry `checkedBy`. OD-149 is `sorted` with a new lean and no `checkedBy`, since its `ifWrong` did not hold. Round 2 of the roadmap, from the evening of 2026-10-02 to past midnight, on the ThinkPad, as nine lanes merged into `feature/round-2`, then an integration, a review and a fold-in. The owner's words at the opening: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." During the round: "we will need a feature branch"; "i agree with the least noisy and cleanest path forward."; "good note on the load. i have moved this to machine that is not capable enough. but i will relocate to M3 mac tomorrow. hopefully we will manage today."; and on 2026-10-03, "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue", "We can discuss details before preparation of work for the next session." and "We will continue development of this project here on the thinkpad. It is not ideal, but sufficient. M3 Mac is continuting with macos native development." Decisions 40 to 45 hold what followed from them. @@ -173,9 +191,28 @@ Twenty-six more findings are `partially-fixed`, each waiting for its slice. Remo ## Next unit -`ROADMAP.md` is the order, ruled as cut by decision 36, and rounds 1 and 2 are done. The next subject is the online rebuild of the order summary, roadmap 3b, with Sc L2-5 at the native tier: package 3b of `design-s4-paper.md` in the unit folder names its files, its order, its proof obligations and what it must not build, and section 7 adds the rulings after the check, `fillTenants` and the resumed batch size among them. The owner said on 2026-10-03: "We can discuss details before preparation of work for the next session." So the main thread lays the next session's work out with the owner before any brief is written or any lane starts. +`ROADMAP.md` is the order, ruled as cut by decision 36, and rounds 1 and 2 are done. The next unit is the online rebuild of the order summary, roadmap 3b, with Sc L2-5 at the native tier, as `layout-3b.md` in the tooling unit's folder lays it out. That means thirteen functions per composition, 33 proof obligations, and three lanes at once (fix, proof, native), then an integrator, a review and a fold-in. Package 3b and section 7 of `design-s4-paper.md` in the round 2 folder remain its Spec source. Its eight questions are ruled as rulings 2 of the tooling unit says: + +- Version 2 goes on the fixture's `documentTitle`, at `convex-test`. +- The renamed example goes on the fixture, with four of its lines changed. +- The journal check is by operation. +- A `CancelOrder` runs on one of the second tenant's 100 seeded orders while the chain is interrupted, and asserts the order's final row `cancelled`. +- The two ledger rows stay `open`, and the fold-in proposes `partially-fixed`. +- The ledger's extra native shares run at `convex-test` only. +- The L2-5 file sets 300 s, beside the test's own 30 s stall bound. +- One 100-batch chain is timed on the quiet machine before the lanes start. + +The five register rows are recorded with new leans, four of them checked (Owner queue). The setup change covers the eight helpers and the test files that call `activate` directly. The factory is not tried. The brief is written at the next session from `layout-3b.md` and rulings 2; the owner's discussion of the layout happened in this unit and ended in "I agree with all recommendations". The main thread ruled at this close that 3b may start: decision 2 asks for no open blocker inside a slice, and the five rows are recorded with leans that keep the Spec text as written, four of them checked, so the brief builds the same text whether or not they are `decided`; they become `decided` when 3b's fold-in applies them, as `SESSIONS.md`, "Who decides", has a `delegated` row ruled by its checked lean. The layout's lane table still says "16 test files" for the native lane's setups, which rulings 2 corrects. + +After 3b, the hosted tier's build, as a unit commissioned from `design-hosted-tier.md` in the tooling unit's folder: its build cut, its order and its "built when" list. The owner's words of 2026-10-03: "Local backend should be primary, IMO. But happy to take the hosted as default if easier for start." So the local backend stays the primary target, and the hosted deployment is a target of the native tier on a schedule, never in `test:all`. Before it starts: + +- The owner creates the GitHub environment `hosted` with its three variables. That means running the commands in the setup record at the owner's own prompt, or granting the agent a permission rule and rerunning the setup from its step 4. +- The owner makes the deploy key `hosted-ci` in the dashboard, with the fourteen actions or, by the CLI, with all of them. +- The owner stores the key with `gh secret set` at an interactive prompt. +- The owner checks the dashboard: one deployment, no production deployment, one key. +- `patch-hosted-specs.diff` is applied after sitting two rules its words. -The second subject of that discussion. The owner asked on 2026-10-03 whether to start de-risking the implementation with tests against a real Convex backend and a minimal setup based on `convex-event-sourcing`, and said design work for tooling could be commissioned separately. The main thread answered that the native tier already runs against the real backend, and that the risk no tier covers is a hosted deployment: F12 and F16 stayed documented and OD-036, Probe 3's quota half, is unrun. It recommended a read-only astra scout over the earlier platform's tooling under the two mining rules (a fact with its date and versions, none of its designs or habits), then a small `build` unit for a hosted tier gated by a deploy key and run on a schedule, with the deployment arrangement designed first. +Its first runs are the owner's from this machine. The four hosted examples run by the workflow once it is on `main`, because GitHub runs a schedule from the default branch. `slice` S3 is round 1's attribution and round 2's measurement, the quiet timings included. S4 is round 2's paper and gate, and the builds of rounds 3 and 4. S5 is round 5. The paper decisions S4's rebuild needs are taken; those of the history view and of S5 are listed below. @@ -210,7 +247,7 @@ What the platform review of 2026-10-02 asks of the later units. The ledger owns ## Tactical decisions taken on 2026-10-01 -The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words. Any of them can be reopened. +The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words, and 46 and 47 during the tooling unit on 2026-10-03. Any of them can be reopened. 1. Slices replace the eight fix units of the previous plan. 2. A slice is built on Specs at `defined` with no open blocker inside the slice. The owner states `ready` after the slice passes. @@ -257,6 +294,8 @@ The owner accepted the first nine on the session's recommendation. The session t 43. One Fable designer took the whole S4 paper, both subjects, because the gate, the rebuild and the operator entries share surfaces, and the owner said "fable only for key design decisions on a larger bit of roadmap". Opus leads, sol and astra did the rest. Fable was spent on the main thread and that one designer. 44. The ThinkPad (Fedora, an Intel i5 with 16 GB) is the development machine for now. The owner's words of 2026-10-03: "We will continue development of this project here on the thinkpad. It is not ideal, but sufficient. M3 Mac is continuting with macos native development." Seven lanes at once drove its load average to 13 to 18 and made native tests time out. So at most three or four lanes that start native backends run at once, a test that fails on a timeout is rerun once alone, and a latency or throughput timing is taken while nothing else runs. 45. An integrator never uses `pkill` on vitest: `pkill -f "vitest run"` matched every lane's run once and interrupted another lane's native file. A run is stopped by its own process ID. +46. When the owner asks for one, a design unit's key decisions go to one Fable advisor, with the owner's principles named in its task: quality, simple design, a maintainable platform. The owner's words of 2026-10-03: "please spawn fable advisor to recommend decisions. let's keep it less noisy and use one advisor. quality, simple design, maitanable platform are guiding principles." An Opus lead turns the memo into the design file and the Spec patch. Sol checks both, and the memo before them, each on a copy with one planted citation. An Opus draft written before the Fable spend was confirmed, such as the hosted lane's `draft-hosted-opus*`, is input, not a decision. With no such request, the standing word holds: "no advisors now". +47. No agent of this harness writes a secret. On 2026-10-03 the main thread's harness refused a secret-store write even through an agent, a "Secret-Store Writes" denial, and the main thread does not work around it. The owner creates a deploy key by hand and stores it at an interactive prompt, never in a chat or in a file in the repository. The setup around the key (the project, the deployment, the GitHub environment and its variables) is an agent's work. The owner's words: "This setup cannot be documented and executed by an agent? With gh cli, convex cli, and other tooling?" and "1. and if someting is an issue, i will jump in." When an agent's own permission check refuses a step, as it refused the setup lane's `gh api` write, the agent stops and hands the owner the exact commands. ## Owner queue @@ -264,12 +303,14 @@ Every open decision is a row of `design/decisions/register.json`, and `python3 d ``` total: 154 -status: unsorted=0, sorted=100, waiting=24, decided=15, folded=15 +status: unsorted=0, sorted=100, waiting=22, decided=17, folded=15 class: tactical=3, delegated=86, owner=50, null=15 advisor: convex=36, domain=45, operator=34, product=39 ``` -Decided since the last close: the twelve `owner` rows of sitting one, recorded as the owner's (decision 40), and OD-007, decided by the convex advisor with its checked lean as the ruling. OD-154, the grant read, is a new `delegated` row with its measured cost. +Decided since the last close: OD-010, installation through the first rebuild, and OD-014, the projection chosen by the generation's version. Both were decided by the domain advisor with their checked leans of 2026-10-02 as the ruling, as rulings 2 of the tooling unit says. The advisor's patch (`patch-register-3b.json`) went in through `decisions.py apply`. OD-057 and OD-080 moved from `waiting` to `sorted`, and OD-057, OD-071, OD-075, OD-080 and OD-149 carry the advisor's new leans. A second patch through the tool set `checkedBy` to `gpt-6.1-sol, medium, 2026-10-03: every checked claim held` on the first four, from sol's recheck. OD-149's stays null: its `ifWrong` says a tenant missing from the list gets no row until `fillTenants` and a rebuild, while a live write in the active generation inserts a missing row (`projection-contract.sdp.md:49`). That changes no `do`, but the register's form for `checkedBy` is "every checked claim held". The hosted patch, when it is applied, adds OD-155, the hosted tier's money, as an `owner` row. + +At the close before, sitting one's twelve `owner` rows were recorded as the owner's (decision 40), and OD-007 was decided by the convex advisor. OD-154, the grant read, is a `delegated` row with its measured cost. Sitting two waits for the owner. Its forks are drafted the same way as sitting one's when the owner asks: @@ -277,12 +318,16 @@ Sitting two waits for the owner. Its forks are drafted the same way as sitting o - `OD-044`, D2's premise, with F18 against it: on the pinned backend one transaction's rows share one commit timestamp in the parent and in components and later transactions get larger ones, while D2 and `spec:context.journal` say no order token for the whole deployment exists. Nothing uses the internal `getSnapshotTs()`. - `OD-063`, OQ1, a component per context, with its local numbers on the lean: a fixture component call cost 3.56 and 5.76 ms over a helper call in two readings under load, and Probe 11 read 7.6 to 15.5 ms per empty component call, all at the native backend tier on the local backend; on the quiet ThinkPad, 3.17 ms per component call over a helper call. - `OD-114`, D13's do-nothing check, with Probe 7's facts: a restore in place keeps schedules the restored data no longer explains, a fresh restore keeps none, and a scan of `_scheduled_functions` stops at about 4,088 outstanding rows on the read limit. -- The language rows, OD-009, OD-022, OD-029, OD-139 and OD-140, with the words the round added: "snapshot" for the exported archive, which `CONTEXT.md` lists to avoid for state; "tenant fill" for `fillTenants`, `fillTenantsBatch` and `tenantFill`; the measurement's words (measurement record, cell, execution against attempt, rerun, contention, controlled change, local backend against hosted deployment); and the words of `ROADMAP.md`'s list. +- The language rows, OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added: "snapshot" for the exported archive, which `CONTEXT.md` lists to avoid for state; "tenant fill" for `fillTenants`, `fillTenantsBatch` and `tenantFill`; the measurement's words (measurement record, cell, execution against attempt, rerun, contention, controlled change, local backend against hosted deployment); and the words of `ROADMAP.md`'s list. The tooling unit adds four sets: + - R5's four, which the unit used as its rulings defined them: local backend, the pinned process this harness starts; hosted deployment, a Convex Cloud deployment; deploy key, a hosted deployment's credential; admin key, the local backend's. + - The hosted patch's new words, which wait for this sitting before the patch is applied: the fields `target`, `statedPlan`, `ranBy`, `hosted` and `windowCrossed`; the types `HostedTarget`, `HostedRunFacts` and `HostedUsage`; the Design key `hostedProbeCopy`; plan and plan class; stated plan; usage reading; kept dataset; planting record; hosted run; preview deployment. Two clashes come with them: "marker", which already names an aggregate's marker, and "pass", which the rebuild's progress row already uses. + - 3b's words, from `layout-3b.md`'s last section: backfill and the backfill mode (which `CONTEXT.md:107` lists to avoid), verify pass, switch, rollback, retired generation, purge, install or first rebuild, fence, batch chain, progress row, pass, the role of a written generation, interrupt, park, stall, tenant list. + - The operator term: `CONTEXT.md:226` makes the operator a kind of actor, while the S4 paper's ruling 4 says no actor of kind `operator` is established and an operator entry takes an `operator` string. Until the owner rules, `CONTEXT.md` is read as ruling 4 says. - OD-008 with OD-006, OD-138, and the S5 rows OD-041 and OD-042, as round 3's sitting two cuts them. -`delegated` rows that wait for the next sort of their lens, each with the round's result on its lean: OD-057 (Probe 6 and the rebuild's own batch driver), OD-066 (3N + 5 with every term attributed; the journal tail read is 10 documents and 6,010 bytes, 29 percent of the documents read on ten lines), OD-120 (Probe 7's states) and OD-154 (3 documents and 918 bytes for the grant read) for the convex lens, and OD-081 (Probe 7's restore answers) for the operator lens. The session may not decide a `delegated` row (`decisions.py`: "session may decide only tactical rows"). +`delegated` rows that wait for the next sort of their lens, each with the round's result on its lean: OD-066 (3N + 5 with every term attributed; the journal tail read is 10 documents and 6,010 bytes, 29 percent of the documents read on ten lines), OD-120 (Probe 7's states) and OD-154 (3 documents and 918 bytes for the grant read) for the convex lens, and OD-081 (Probe 7's restore answers) for the operator lens. OD-057 has left this list: its new lean closes on Probe 6's result and is checked. The session may not decide a `delegated` row (`decisions.py`: "session may decide only tactical rows"). -The 86 `delegated` rows carry the leans of the first panel run. A `gpt-6.1-sol` check read the first 23: twelve held on every checked claim and carry `checkedBy`; eight hold with a gap and three do not hold (OD-066, OD-103, OD-109), each waiting for its advisor's correction with the check's findings in the unit folder the project notes name; the later leans are unchecked. A checked lean is applied to the Specs as a ruling by a later unit, one Spec family at a time, and the row becomes `decided`. The rest carry a one-line lean or wait for a probe, a measurement or a build (`--waiting`). +The 86 `delegated` rows carry the leans of the first panel run, five of them replaced in the tooling unit. A `gpt-6.1-sol` check read the first 23: twelve held on every checked claim and carry `checkedBy`; eight hold with a gap and three do not hold (OD-066, OD-103, OD-109), each waiting for its advisor's correction with the check's findings in the unit folder the project notes name. The tooling unit's recheck adds four rows that carry `checkedBy` (OD-057, OD-071, OD-075, OD-080), so sixteen carry it, OD-010 and OD-014 among them, now decided. The other leans are unchecked. A checked lean is applied to the Specs as a ruling by a later unit, one Spec family at a time, and the row becomes `decided`. The rest carry a one-line lean or wait for a probe, a measurement or a build (`--waiting`). ## Leads @@ -363,6 +408,38 @@ A session confirms or drops each. None is a finding yet. - From the S4 paper: E-48 and E-49 are numbers of package D's range on a package C Spec, because C's range is full; the journal's baseline driver is not written as an operator entry, and the baseline writer changes sources without maintaining read models; the receipts sweep loops with `nextTenant`, which `receipt-table.sdp.md` does not say yet; a scheduled mutation that loses a race is assumed to rerun until it commits, and no pinned package shows a bound. - From the probes, not run: Probe 6's timeout, retry exhaustion, a restore with a component worker pending and `parallelize`; Probe 7's hosted dashboard restore, the seven-day expiry, Workpool and Workflow (OD-115), file storage, a schema change between export and import, the Layer 2 restore drill, the argument limits of an action, and the limit of one million outstanding schedules. +Leads 30 to 38 are lane data's of the tooling unit, as `leads-data.md` in its folder words them. + +30. The harness keeps a local backend's stdout and stderr only in a redacted rolling buffer of 16,000 characters, used to explain a failed start or a taken port, and the run record holds test errors and explicit measurements but no backend output (`harness/backend.ts:87,102-132,138-161`, `harness/evidence.ts:21-58,115-130`). A deliberately failed native case, followed by a read of what its record kept, would settle whether the backend's output must be kept beside the record. + +31. A run record holds no memory, CPU or other resource sample of the local backend process unless a test stores one as a measurement. One passing and one failing native case that sample the owned process would settle which samples explain a failure and whether taking them is worth its cost. + +32. No authored file under `fixture/`, `example/` or `tests/` carries the `"use node"` directive (the only matches are comments in generated files), so no native test runs a Node action or its calls back into the backend. Probe 14's Node-action example would settle it on the pinned release, with the Node executable and version recorded. + +33. The harness spawns the local backend with no `env` option (`harness/backend.ts:105-124`), so the backend inherits the test process's whole environment, and the pinned release reads each of its defaults through `env_config`, which takes a variable of the same name from that environment and logs the override only at info level (`crates/cmd_util/src/env.rs:23-48` at `5c7cb5b`): a `MAX_BYTES_WRITTEN_PER_SECOND` or `SCHEDULED_JOB_EXECUTION_PARALLELISM` in the shell changes the backend, and neither `facts().environment` (`:293-306`, which lists the deployment variables the harness set) nor the record shows it, since the output is not kept (lead 30). Starting the backend with an environment of its own, as every CLI call already gets, or recording the variables it inherited that name a default, checked by a native case that supplies one override on purpose, would settle it. + +34. The design does not take these shapes, named by kind: fixed slots and machine-wide leases on shared backends, quarantine and capacity registries, accounting over a whole container engine's volumes, nested ownership of cleanup, whole-suite retries against kept state, a passing foreground suite taken as proof that background work finished, test-mode tuning taken as production evidence, a shared backend's timings taken as a portable limit, copied checkouts and conversion caches, preview provisioning times claimed without a measurement, and a container's start time or port behavior taken as the binary's. A measured gap of the native tier, or a measured need of a hosted deployment, with a bounded alternative and its cost of ownership, would be the reason to reopen one of them. + +35. The pinned release marks a failed clean-isolate check as `IsolateNotClean`, a refusal before execution that its own comment calls retryable (`crates/isolate/src/isolate.rs:326-335` at `5c7cb5b`), and `ConvexHttpClient` throws such a refusal to its caller with no retry (`sourceHttpMutationFailure` of the harness Spec), so one such refusal ends a native fixture's call; the earlier platform met it with `convex` 1.42.1 and again on 2026-09-26 with `convex` 1.46.0, and it has not been seen here. A native case that provokes the condition, binds that refusal, shows no write stored and shows the same mutation succeed on the same deployment would settle how often it happens and what the client gets; no cheap way to provoke it is known, which is why it is a lead and not a probe. + +36. The pinned release hands one configured origin to file storage and to Node actions (`sourceBackendOrigin`), and the native tier has shown neither a generated storage URL nor a Node callback reaching it. Probe 14's two examples would settle both for the harness's direct process; they say nothing of a container's addresses or of a hosted deployment. + +37. What a failed native case leaves after teardown is known only from the cleanup source: storage is removed on disposal, and a backend whose process cannot be inspected keeps its directory and fails the sweep. A deliberately failed case, followed by a read of its record, its directories and the process table, would settle what teardown leaves behind. + +38. The timings the earlier platform took on 2026-09-26 (`convex` 1.46.0, a Docker backend) say nothing about the pinned release here, and F19 already records this release's system-operation refusal apart from the one-second execution limit. A reproduction under a recorded environment and load would settle any remaining question about time, and it would record times, not assert a call count or a start time as a limit. + +39. From the hosted lane and its setup. Each item rests on source, docs or the setup run; no hosted test has run. + - Whether the table `plantedSchedules` of the hosted copy keeps its documents when the production composition, whose schema leaves it out, is deployed between two runs. No source in the hosted report shows it. One local run settles it: the proposed example `native-harness.undeclared-table-survives-another-composition-deploy`. If the rows are lost, the acceptance example deploys a copy of the production composition that declares the table. + - `--expiration none` for a development deployment: settled. The setup lane's `deployment create` with CLI 1.46.0 accepted it on 2026-10-03. + - The deployment URL. The CLI printed `https://wandering-mule-403.eu-west-1.convex.cloud`, with the region, while the hosted patch's rule refuses a URL that is not `https://`, the key's deployment name and `.convex.cloud`. Either the rule takes the printed form, or a run shows the short form answers. The setup did not try the short form. + - Serial runs are not enforced. A GitHub concurrency group does not serialize a developer's run against CI's, and one machine cannot see another's run. So the record names who ran it, `ranBy`, and a difference of two usage reports is the run's own only when no other run overlapped it. + - Redaction misses two paths until the build adds them: the HTTP and socket clients built with no logger (`harness/admin.ts`), and an aborted child call, which rejects with its abort reason unredacted (`harness/child.ts:83`). + - That a development deploy key with the fourteen actions covers every CLI call the harness makes, export and import included, is from source only; the first hosted run settles it. +40. From the check of the 3b memo, beside what rulings 2 took. + - `fixture/convex/migrations.ts:51` spreads the projection's result after a `!`. TypeScript erases the `!`, so a `null` fails at the write and not there. Line 39 is a `typeof` type query, so the rename there changes a type expression too. + - Test files call `activate` directly, not only through the eight helpers: `tests/native/production-orders.test.ts:50`, `tests/native/read-models.test.ts:82`, `tests/simulator/read-model.test.ts:220` and on. 3b's setup change edits them. + - `restore.sdp.md:82` does not say that history arrives by import, which the memo's question 8 read into it. + The reports behind leads 1 to 3 and 6 are outside the repository, because they quote a private repository. The project notes file named in `AGENTS.md` lists them, with the reports, briefs and rulings of S0. ## Inputs diff --git a/design/decisions/register.json b/design/decisions/register.json index 1f1d993..1cf4705 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -374,7 +374,7 @@ "provisionalReading": "Extension E-8, the first generation: the doc says how a generation is rebuilt and not how a read model's first generation comes to exist; the option taken here is one explicit transition, the first activation, run with admin access before the first command that writes the read model, legal only when the read model has no generation row, with a command that finds no generation to write failing as a technical failure and a query that finds no active generation failing the same way; the owner has not ruled (E-8, D9, D19)", "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "delegated", "reason": "Rule 3, which mechanism: the doc names no path for a first generation [design/specs/application/generation-registry.sdp.md:36], and an edit to one Spec and one file takes the path back; nothing here edits the doc or trades values without evidence.", @@ -388,9 +388,9 @@ }, "checkedBy": "gpt-6.1-sol, medium, 2026-10-02: every checked claim held", "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "advisor:domain", + "decidedOn": "2026-10-03", + "ruling": "Install every read model through its first rebuild: generation 1 starts as `building` through `startGeneration`, its backfill covers whatever subjects exist, and the switch makes it active; no transition leads from nothing to active. The checked lean of 2026-10-02 is the ruling, and the Specs carry it [design/specs/application/generation-registry.sdp.md:46,49]." }, { "id": "OD-011", @@ -548,7 +548,7 @@ "provisionalReading": "Extension E-9, two projection versions: a read model holds one projection, so `applyProjection` writes every generation it is given with that projection, whatever `projectionVersion` the generation records; after a projection version change the active generation was built by the earlier version, and the owner has not ruled how a command writes two projection versions at once, the active generation's and a building generation's (E-9, D8, D9)", "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "delegated", "reason": "Rule 3, a signature and the order of steps: the doc says live update and rebuild share one projection and fixes no shape [design/specs/application/projection-contract.sdp.md:33]; an edit to one Spec and two files takes it back.", @@ -562,9 +562,9 @@ }, "checkedBy": "gpt-6.1-sol, medium, 2026-10-02: every checked claim held", "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "advisor:domain", + "decidedOn": "2026-10-03", + "ruling": "A read model declares its projection versions as a list, the latest first, and `applyProjection` writes each generation with the projection whose version the generation row records, failing as a technical failure when no declared projection carries that version. The checked lean of 2026-10-02 is the ruling, and the Specs carry it [design/specs/application/projection-contract.sdp.md:47,67,68,90]." }, { "id": "OD-015", @@ -2229,19 +2229,19 @@ "provisionalReading": null, "blocks": "S4", "advisor": "convex", - "status": "waiting", + "status": "sorted", "foldedInto": [], "class": "delegated", "reason": "Rule 3: which Convex mechanism drives a backfill batch, settled by a probe and taken back by an edit [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:22; design/README.md:219].", "lean": { - "do": "Run Probe 6 in S4 before the rebuild is built on the component, and until it passes keep E-40's split: @convex-dev/migrations for parent-table batches and a self-scheduled internal mutation for context-enumerated ones.", - "restsOn": "F17 is assumed: the component's page states batches with saved progress and resume, and not batch size or a race with a live command [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:13,29]. Probe 6 must show a backfill batch racing a live command under optimistic concurrency, resume after interruption, and the component driving a batch whose source is a context's enumeration [design/specs/facts/probe-plan.sdp.md:63; design/specs/application/rebuild.sdp.md:57]. E-40 already carries the split until Probe 6 shows the component can [design/README.md:219]. Mutations are serializable under optimistic concurrency, the assumption the race rests on [design/specs/context/persistence-adapter.sdp.md:38]. **Judgment.** The component buys checkpointing and resume that a generation row and a self-scheduled mutation can also keep; a mechanism that restates what a scheduled mutation gives needs the probe to earn its dependency. Probe 6 ran, at the native backend tier on the pinned release: @convex-dev/migrations 0.3.6 walks only the tables of the component it is defined in, drives a context's rows only through a parent internal mutation written by hand against a contract its README does not document and the compiled tier needs a cast for, keeps its own status apart from the generation's state, and leaves no error on its row when a batch fails past the read limit [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:35,44,49]. So every batch, backfill, verify and purge alike, is driven by the platform's own self-scheduled internal mutation, the rebuild uses the component for nothing, and the progress row is the one checkpoint [design/specs/application/rebuild.sdp.md:58,86,129; design/specs/decisions/d09-rebuild-online-by-default.sdp.md:42]. The lean's condition was met in a way it did not foresee: the split it keeps until the probe passes is gone, because the probe showed the component does not fit.", - "against": "The do-nothing option is no component at all: one self-scheduled internal mutation for every batch, no new dependency and one driver instead of two; the leads name two checkpoint owners for migrations-driven batches as a hazard [design/STATE.md:276, a pointer]. If Probe 6 passes, the split costs a second driver for nothing.", - "settles": "Probe 6 on the native backend at the pinned release, precompiled-2026-09-28-5c7cb5b with convex 1.46.0, in S4: a batch racing a live command on one read-model row, interruption and resume, and a batch over a context's enumeration, recorded as a run record and on F17.", - "changes": "spec:facts.f17-migrations-fits-generation-backfill (status), spec:facts.probe-plan, spec:application.rebuild, spec:decisions.d09-rebuild-online-by-default; S4. The alternative, no component, edits spec:application.rebuild and the E-40 row and removes a dependency.", - "ifWrong": "If the component is kept and does not fit, the parent-table driver is swapped for the self-scheduled one, an edit to one Spec and one driver; the cursor and the batch bound are the design's numbers, not the component's [design/README.md:219]." + "do": "Close on Probe 6's result: the rebuild drives every batch, backfill, verify and purge, with its own self-scheduled internal mutation, uses `@convex-dev/migrations` for nothing, keeps the progress row as the one checkpoint, and keeps the component pinned as a dev dependency for the fixture's probe only.", + "restsOn": "Lean of the domain lens, under the owner's one-advisor word of 2026-10-03, on a row whose advisor is `convex`. Probe 6 ran at the native backend tier on `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, `convex-helpers` 0.1.124 and `@convex-dev/migrations` 0.3.6 [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:13], and F17 is probed [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:11]. On those pins the component reaches a context's rows only through a parent mutation written by hand against a contract its README does not document and the compiled tier needs a cast for, its status and the generation's state are two vocabularies, and a failure past the read limit leaves no error on its row [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:21,44,45,49]. The Specs already take the consequence: every batch is driven by a self-scheduled internal mutation under one batch contract and the rebuild uses the component for nothing [design/specs/application/rebuild.sdp.md:58,59,86; design/specs/decisions/d09-rebuild-online-by-default.sdp.md:42]. The round 2 close records the pin as a dev dependency for the fixture's probe only [design/STATE.md:38, a pointer]. **Judgment.** The earlier lean kept a split of two drivers \"until it passes\"; the probe did not pass the component, so that condition was not met but replaced: the split is gone and one driver remains, the do-nothing option the earlier `against` named. Nothing in this row changes a Spec: the ruling records what the Specs say.", + "against": "**Judgment.** The component gives checkpointing, resume and a status for free where it fits, and a later release might fit; the probe shows 0.3.6 on the pinned backend and nothing about a later one. The cost of the Spec's choice is that the platform owns its batch driver and its stall answer, which is `abortGeneration` and a new `startGeneration` [design/specs/application/rebuild.sdp.md:144].", + "settles": "Settled by Probe 6 at the native backend tier on the pins named; what the six examples do not run, hosted quotas, sustained contention, automatic stall detection, timeout and retry exhaustion [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:31,49], stays open on F17 and is not this row's.", + "changes": "No Spec: `spec:application.rebuild`, `spec:decisions.d09-rebuild-online-by-default` and `spec:facts.f17-migrations-fits-generation-backfill` already carry the ruling. The register row alone. The alternative, a driver on a later release of the component, would edit `spec:application.rebuild` at its E-40 question and `fnBackfillBatch` and add a dependency.", + "ifWrong": "A later release that fits is a new fact with its own probe; the batch contract is one internal mutation per batch [design/specs/application/rebuild.sdp.md:111,114], so a driver swap is an edit to the rebuild Spec and the batch functions, and the cursor and the bound are the design's numbers either way [design/specs/application/rebuild.sdp.md:88,146]." }, - "checkedBy": "gpt-6.1-sol, medium, 2026-10-02: every checked claim held", + "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held", "fork": null, "decidedBy": null, "decidedOn": null, @@ -2824,9 +2824,14 @@ "class": "delegated", "reason": "Rule 3, the shape of a table and which row holds a cursor; the doc binds batching to the component and fixes no checkpoint [design/specs/application/generation-registry.sdp.md:37]; an edit to one Spec and one file takes it back. Doubt: the lean goes against the built reading of [design/specs/application/generation-registry.sdp.md:47] on a contention judgment that only the native tier shows, and S4's guidance [design/STATE.md:160], a pointer, asks for progress away from rows every command reads.", "lean": { - "do": "Move the backfill and verify cursors and the counts to a progress row of their own that only the batch reads, keep state, projection version and fence on the generation row every command reads through `by_read_model_state` [design/specs/application/generation-registry.sdp.md:67], and treat the migrations component's own state as derived from the progress row." + "do": "Keep the two rows as the Specs pin them: the generation row carries state, projection version, fence and who started and last changed it, and a batch patches it only in the mutation that ends its pass; the progress row carries the cursor, the batch size and the counts and is the one checkpoint of every batch; no state of the migrations component exists beside them.", + "restsOn": "The E-40 question takes the progress row because a batch that wrote its checkpoint on the row every command reads would re-run every command in flight once per batch [design/specs/application/generation-registry.sdp.md:38]; the generation row carries nothing a batch writes on its healthy path except the state the batch ending a pass sets, and the progress row is written in the same mutation as the rows [design/specs/application/generation-registry.sdp.md:53,74]; `generationsToWrite` and `activeGeneration` read the generation row through `by_read_model_state` and never the progress row [design/specs/application/generation-registry.sdp.md:82,83,94]; a build costs the commands one re-run at the end of the backfill, one at the end of the verify pass and one per operator entry [design/specs/application/generation-registry.sdp.md:86]; the rebuild uses `@convex-dev/migrations` for nothing [design/specs/application/rebuild.sdp.md:86], so the earlier lean's clause that derived the component's state from the progress row names nothing. **Judgment.** One checkpoint with one owner is the rule of my lens: the progress row is the source of where a pass is, the generation row the source of what a generation is, and neither is computed a second way.", + "against": "**Judgment.** Two rows cost a second table, a second read per batch and a second write at the end of a pass; and whether a batch's checkpoint re-runs no command is shown only at the native tier, since `convex-test` orders nothing concurrently. The ledger still asks for a contention run against a run with no rebuild [design/reviews/consensus-ledger.json:3946].", + "settles": "The `convex-test` obligation that the generation row is untouched by a batch except at the end of a pass (`design-s4-paper.md:104` in the round 2 folder) and the native L2-5 example's required assertions that no command failed for reading the registry while batches checkpointed and that the generation rows changed only at the start, the end of each pass and each operator entry [design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md:36]; neither has run. The comparison run is a later measurement.", + "changes": "No Spec: `spec:application.generation-registry` `tableGenerations`, `tableGenerationProgress` and `transitionCost` and `spec:application.rebuild` `step1`, `step6` and `occRace` carry it. The register row alone. The alternative, one row, would edit the same entries and the ledger's checkpoint finding.", + "ifWrong": "If the registry read set of a command still meets a batch's write, each pass end re-runs commands once, which `transitionCost` already states [design/specs/application/generation-registry.sdp.md:86]; a meeting beyond that would be found by a contention run that records each refused execution's `occInfo` tables as the measurement does [design/specs/application/first-experiment.sdp.md:114], and an edit to the index use or the row split takes it back." }, - "checkedBy": null, + "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held", "fork": null, "decidedBy": null, "decidedOn": null, @@ -2990,9 +2995,14 @@ "class": "delegated", "reason": "Rule 3, signatures and the shape of a row: the doc gives no signature, row shape or input for the three forms [design/specs/application/projection-contract.sdp.md:33]; an edit to one Spec and the read-model files takes it back. The order in which the forms are bound is the main thread's [design/STATE.md:160], a pointer.", "lean": { - "do": "Keep the `ReadModel` and `Projection` split, the write by generation role and the row measured against its budget at every write, and bind the aggregate and history forms only at a view that needs them." + "do": "Keep the per-entity form as the Specs pin it: a `ReadModel` with its `projections`, the latest first, and a `Projection` of `version`, `keyOf` and `project` alone; `applyProjection` writing each generation by its role, `active`, `building` or `verifying`, with the projection whose version the generation records; and every row measured against `rowBudgetBytes` at every write. The history form stays as declared and bound for the order allocation history, whose build is not 3b's, and the aggregate form stays a signature with no read model of the example bound to it.", + "restsOn": "The E-9 question names the three forms and the three points the owner has not ruled, the split, the write by role and the measured row [design/specs/application/projection-contract.sdp.md:33]; the declaration and the helper are pinned [design/specs/application/projection-contract.sdp.md:47,66,67,68]; the write rule by mode and role is pinned, live-updated in the role `building` skipping a missing row because the backfill has not reached it, and a missing row in `verifying` meaning a subject that projected to none [design/specs/application/projection-contract.sdp.md:50,69,92; design/specs/application/rebuild.sdp.md:69,70]; the row is measured with `getConvexSize` at every write [design/specs/application/projection-contract.sdp.md:51]. The history form is declared as `orderAllocationHistory` [design/specs/application/projection-contract.sdp.md:78] and bound once per source stream type in the example [design/specs/application/orders-inventory-example.sdp.md:69], and 3b does not build the paused path (`design-s4-paper.md:107` in the round 2 folder); the aggregate form's table is given as a shape and not as a read model the first experiment builds [design/specs/application/projection-contract.sdp.md:85], and the online rebuild is pinned for a per-entity read model with one source stream type while an aggregate gets a rebuild when a view needs it [design/specs/application/rebuild.sdp.md:61]. **Judgment.** The roles exist because the building generation has rows the backfill has not reached and the verifying one does not; one rule for both would read a missing row two ways.", + "against": "**Judgment.** The write rule varies by mode, by role and by the row's state, with the `null` candidate beside them, so its pure table is wide (`design-s4-paper.md:103,144` in the round 2 folder), where a rule without roles would be narrower; it is not taken because it cannot tell a row the backfill has not written from a subject that projects to none [design/specs/application/rebuild.sdp.md:70]. Measuring every row at every write costs a `getConvexSize` per write.", + "settles": "The pure write-rule table of mode by role by row state, with the `null` row (`design-s4-paper.md:103,144`), and the `convex-test` obligations of a live create and a live update against `building` and against `verifying` and of two projection versions at once (`design-s4-paper.md:104`), none yet run; the owner's confirmation of the three forms, which the E-9 question asks for, stays the owner's and this ruling does not take it.", + "changes": "No Spec: `spec:application.projection-contract` `typeReadModel`, `typeProjection`, `typeWritableGeneration`, `step1` to `step4` and `limitRowSize` carry the per-entity rules, and `tableOrderAllocationHistory` and the example's binding carry the history form. The register row alone. The alternative, one write rule without roles, would edit `step3` and the rebuild's rule on a missing row in `building`.", + "ifWrong": "A signature change to `Projection` or `ReadModel` is an edit to the projection contract and the compositions' read-model files; every row carries its `projectionVersion` [design/specs/application/projection-contract.sdp.md:44] and every version writes rows the one table's validator accepts [design/specs/application/projection-contract.sdp.md:47], so stored rows keep their meaning across it; a role added or removed changes `step3` and the pure table." }, - "checkedBy": null, + "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held", "fork": null, "decidedBy": null, "decidedOn": null, @@ -3210,15 +3220,19 @@ "provisionalReading": "The option taken here: a batch is one mutation with one cursor and a bound of at most 100 entities, capped twice, by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget, a design choice under F13 that is a field of the generation row and that `resumeGeneration` lowers down to 1 when a generation stalls on optimistic-concurrency conflicts over a hot key range, because a mutation that loses the race never commits and cannot count its own losses; the migrations component drives batches whose source is a parent table, and a self-scheduled internal mutation with the same batch contract drives batches whose source is a context's enumeration until Probe 6 shows the component can; Probe 6 also measures the contention and decides whether an automatic stall check earns its function (E-40, E-24, E-9, F17, F13, F1)", "blocks": "S4", "advisor": "domain", - "status": "waiting", + "status": "sorted", "foldedInto": [], "class": "delegated", "reason": "Rule 3, a bound and its number and which mechanism drives a batch: the doc gives no batch size [design/specs/application/rebuild.sdp.md:58], and the Spec itself leaves the component's reach to Probe 6, so a probe comes first.", "lean": { - "do": "Keep the batch of at most 100 entities capped by the row budget and the context's page bound, and the self-scheduled mutation for context-sourced batches until Probe 6 shows the component can drive them.", - "settles": "Probe 6 at the native tier: whether `@convex-dev/migrations` can drive a batch whose source is a context's enumeration, and the contention measured over a hot key range." + "do": "Keep the batch bound as the Specs pin it: at most `rebuildBatchCeiling`, 100 entities capped by the row budget, and never more than the context's `list` page admits, a field of the progress row that `startGeneration` sets and nothing changes after, so every pass and every resumed pass runs at it; every batch, over a context's enumeration or a parent table, driven by the platform's own self-scheduled internal mutation; a stalled pass answered by `abortGeneration` and a new `startGeneration` with a smaller size, and no automatic stall check.", + "restsOn": "The bound and its two caps [design/specs/application/rebuild.sdp.md:88,116,146], the purge cap [design/specs/application/rebuild.sdp.md:147], the one driver for every batch [design/specs/application/rebuild.sdp.md:59,86], `resumeGeneration` taking no batch size [design/specs/application/rebuild.sdp.md:132; design/specs/application/generation-registry.sdp.md:46] and the stall answer [design/specs/application/rebuild.sdp.md:144]; the main thread's ruling 9 after the paper's check fixed the resumed size (`design-s4-paper.md:150` in the round 2 folder), and the paper excludes an automatic stall check from 3b (`design-s4-paper.md:107`). Probe 6 settled the driver: the component walks only the tables of the component it is defined in [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:21], so the row's provisional reading, which had the component drive parent-table batches and `resumeGeneration` lower the size, is superseded on both points. **Judgment.** A size that never changes makes the progress row the one source of a pass's shape and the batch count a function of the data, and a stall that needs a smaller size is a new generation, not a mutated one.", + "against": "**Judgment.** A fixed size means a hot key range forces an abort and a whole new backfill rather than a slower continuation of the same generation, and the numbers 100, 16 and `limitPurgeBatch` are design choices under F13 that no run has pressed. Sustained contention over a hot range is not run [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:31].", + "settles": "Probe 6 settled the driver at the native backend tier; the pure test of `batchSizeFor` at 0, -1, 1.5, above the ceiling and at both row budgets (`design-s4-paper.md:103`) settles the bound's arithmetic once run; the native L2-5 example specifies generation 2 at batch size 1 [design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md:33] and has not run; contention over a hot range is a later measurement on the quiet machine.", + "changes": "No Spec: `spec:application.rebuild` `fnBatchSizeFor`, `limitBatchSize`, `limitPurgeBatch`, `stallSignal` and the E-40 question carry it. The register row alone; its `provisionalReading` text is stale and the ruling supersedes it. The alternative, a size an operator lowers on resume, would edit `fnResumeChain`, `fnResumeGeneration` and the transition Contract bullet of the registry.", + "ifWrong": "The ceiling and the caps are constants an edit changes [design/specs/application/rebuild.sdp.md:146,147]; a resumed size would be one optional argument on `resumeGeneration` and one patch of the progress row, and no stored row changes meaning." }, - "checkedBy": null, + "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held", "fork": null, "decidedBy": null, "decidedOn": null, @@ -6131,12 +6145,12 @@ "class": "delegated", "reason": "Delegated test: the order of steps of an enumeration and what it records, inside what D9 and Law 11 rule; no Spec under `design/specs` names where the application's tenant list comes from, which the lean names as the gap.", "lean": { - "do": "Keep the live path as what covers a tenant created after `startGeneration`, and make every deployment-wide pass record the tenants it enumerated on its row, so a tenant it never examined is visible and completion is evidence rather than a count.", - "restsOn": "[design/specs/application/generation-registry.sdp.md:45] makes every command that maintains a read model write the building generation too, so a tenant whose first event comes after the generation row exists has no event the backfill can miss; [design/specs/application/rebuild.sdp.md:104] moves the cursor through the tenant list in tenant-ID order, so a tenant whose ID sorts behind the cursor is skipped by the backfill and the verify pass; [design/specs/application/restore.sdp.md:96] iterates a check under `all` the same way; [design/specs/application/restore.sdp.md:82] shows that history also arrives by `imports` rows. **Judgment.** The tenant that matters is not the one created by commands, which the live path covers, but one whose history arrives without commands, an import behind the cursor, and a check that reports clean for a tenant it never examined; the person who accepts a restore needs the list of what was examined, not a count of batches.", - "against": "Whether a new tenant sorts behind the cursor depends on how tenant IDs are assigned, which no Spec I opened pins, and no Spec I opened says what an import does to a building generation, so the hole may be narrower or wider than this reading; recording enumerated tenants on the row costs one array that grows with the tenant count against the document limit of F13.", - "settles": "The S4 build at the `convex-test` tier: a tenant created mid-pass with commands and one imported behind the cursor, each followed by the verify pass; no owner's word is needed.", - "changes": "For the lean: `spec:application.rebuild` at [design/specs/application/rebuild.sdp.md:104] and `spec:application.restore` at [design/specs/application/restore.sdp.md:96] gain the enumerated-tenants record, and one Spec must name the tenant list's source. For the alternative, a snapshot of the tenant list taken at `startGeneration` into the cursor [design/specs/application/generation-registry.sdp.md:62]: the same two lines, and a tenant created after the snapshot is skipped by rule rather than by sort order.", - "ifWrong": "A tenant missed on a per-entity view is served rows only for subjects written since the generation started, found by a verify pass that enumerates it, which is the circularity the record breaks; the cost is one tenant's stale read model until a rebuild, and an edit to the enumeration takes it back." + "do": "Keep the Specs as written: a tenant whose row enters the list behind a backfill's cursor is not visited by that pass and needs no visit, the verify pass starts again at the first tenant, a tenant fill and a generation in flight exclude each other, an operator runs `fillTenants` before the first `startGeneration` on a deployment whose grants predate the list, and no pass records the tenants it enumerated.", + "restsOn": "Lean of the domain lens, under the owner's one-advisor word of 2026-10-03, on a row whose advisor is `operator`. The tenant list is the `tenants` table, and `insertGrant` inserts a tenant's row when it finds none, in the same mutation as the grant [design/specs/command/actor-and-scope.sdp.md:35,52,77]; a command names its tenant and is authorized by a grant read in the same transaction [CONTEXT.md:119-121,127-129], so on a deployment built with the list a tenant's row precedes its first command and its first stream; a tenant behind the cursor therefore has only streams created after the generation was registered, which the live path wrote [design/specs/application/rebuild.sdp.md:81,92]; the verify pass enumerates from the first tenant [design/specs/application/rebuild.sdp.md:81,95,126,127]; an import is a sequence of bounded batch commands with their own receipts [design/specs/application/parent-use-cases.sdp.md:81], so imported history reaches a building generation through step 9 like any command's. The legacy case: `startGeneration` refuses an empty list beside a grant, `fillTenants` fills the list from the grants, and the two exclude each other [design/specs/application/rebuild.sdp.md:82,83,84]; a partly filled legacy list, one holding a tenant whose first grant came after the table and missing an older one, is not detected by that refusal, so the operator runs `fillTenants` on such a deployment before its first `startGeneration` [design/specs/application/rebuild.sdp.md:62]. The earlier lean's `against` named tenant-ID order and an import's effect as its uncertainties [design/decisions/register.json:6136]; both are answered by the lines above. **Judgment.** A pass visits every tenant that is in the list when its cursor reaches that tenant's position, and a tenant inserted behind the cursor needs no visit because all its streams were written live; the enumerated-tenants record would be a second truth of coverage beside the tenant list and the cursor, growing with the tenant count on a row every batch writes, and coverage here is a consequence of rules, which is the simpler design.", + "against": "**Judgment.** A writer that changes a source without a command, the journal's baseline driver, maintains no read model (`design-s4-paper.md:126`, lead 1, in the round 2 folder); a baseline is an event at the next stream version [design/specs/context/journal.sdp.md:64,100], so the verify pass repairs the row when it examines that source after the baseline [design/specs/application/rebuild.sdp.md:95,127], and a baseline written after that visit, or a tenant absent from the list for a reason not named here, a row deleted or a partly filled legacy list the operator did not fill, is repaired by no pass and shown by no record. An operator who accepts a restore reads counts, not a list of what was examined.", + "settles": "The `convex-test` obligations of a tenant inserted behind the cursor and of the fill's two mutual refusals (`design-s4-paper.md:104,149`), and the native L2-5 example's two tenants [design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md:33], none yet run; the baseline driver and read models is the journal Spec's open item, not this row's.", + "changes": "No Spec: `spec:application.rebuild` at its tenant rules and `spec:application.generation-registry` at its `startGeneration` Contract bullet carry it. The register row alone. The alternative, the enumerated-tenants record, would add a field to `tableGenerationProgress` and to the restore's check scope [design/specs/application/restore.sdp.md:96] and a growth bound under F13.", + "ifWrong": "A tenant with streams older than a generation and no row in the list when a pass reached its position is served no row of the new generation until `fillTenants` and a rebuild, one tenant's stale read model, found when its queries answer no rows; a count of tenants visited on the progress row, or the record the earlier lean asked for, is a field added by an edit and changes no stored row's meaning." }, "checkedBy": null, "fork": null, From 62b8817cef7c71cb14d3af2c3e0c5de8a870f7de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 03:19:39 +0200 Subject: [PATCH 075/155] design: the owner's hosted setup recorded in STATE.md Co-Authored-By: Claude Fable 5.1 --- design/STATE.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/design/STATE.md b/design/STATE.md index 296f12f..650aa47 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -206,10 +206,8 @@ The five register rows are recorded with new leans, four of them checked (Owner After 3b, the hosted tier's build, as a unit commissioned from `design-hosted-tier.md` in the tooling unit's folder: its build cut, its order and its "built when" list. The owner's words of 2026-10-03: "Local backend should be primary, IMO. But happy to take the hosted as default if easier for start." So the local backend stays the primary target, and the hosted deployment is a target of the native tier on a schedule, never in `test:all`. Before it starts: -- The owner creates the GitHub environment `hosted` with its three variables. That means running the commands in the setup record at the owner's own prompt, or granting the agent a permission rule and rerunning the setup from its step 4. -- The owner makes the deploy key `hosted-ci` in the dashboard, with the fourteen actions or, by the CLI, with all of them. -- The owner stores the key with `gh secret set` at an interactive prompt. -- The owner checks the dashboard: one deployment, no production deployment, one key. +- Done by the owner at the owner's own prompt on 2026-10-03, after the close commit: the GitHub environment `hosted` with its `main` branch policy and the three variables (`HOSTED_DEPLOYMENT_URL` as the printed URL with the region, `HOSTED_DEPLOY_KEY_NAME` as `hosted-ci`, `HOSTED_PLAN` as `Professional`), the deploy key `hosted-ci` made in the dashboard on `wandering-mule-403` (its value begins `dev:wandering-mule-403|`, the owner's reading), and the secret `HOSTED_DEPLOY_KEY` in that environment. `gh secret list --env hosted` and `gh variable list --env hosted` showed them. +- Still the owner's: the dashboard check. The project showed a second development deployment, `clean-capybara-16`, which no agent created and which is most likely the owner's personal dev deployment from the dashboard's first open; if nothing uses it, the owner deletes it so that the project holds `dev/hosted` only, no production deployment and one key. - `patch-hosted-specs.diff` is applied after sitting two rules its words. Its first runs are the owner's from this machine. The four hosted examples run by the workflow once it is on `main`, because GitHub runs a schedule from the default branch. From 2b6b9db26a44e948590dc9f1cc6cb491ff761d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 03:49:24 +0200 Subject: [PATCH 076/155] design: 3b the Spec text the proof and the fix share The determinacy pass over the rebuild, the generation registry, the projection contract, the L2-5 example and the fixture's read models: documentTitle's version 2 on the fixture, the write rule's reading of older, the progress row's updatedAt, the exports beside the helpers, the row patches at each entry and pass end, and the L2-5 example as the owner's rulings sharpen it. Co-Authored-By: Claude Fable 5.1 --- .../application/generation-registry.sdp.md | 4 +-- .../application/projection-contract.sdp.md | 4 +-- ...ild.online-rebuild-interrupt-resume.sdp.md | 12 ++++---- design/specs/application/rebuild.sdp.md | 28 +++++++++---------- design/specs/platform/native-harness.sdp.md | 2 +- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/design/specs/application/generation-registry.sdp.md b/design/specs/application/generation-registry.sdp.md index ac58186..1bad472 100644 --- a/design/specs/application/generation-registry.sdp.md +++ b/design/specs/application/generation-registry.sdp.md @@ -64,14 +64,14 @@ The registry is two parent tables. The `generations` table records every generat Tables live in the parent's `schema.ts`, spread from the library's `readModelTables`. `streamVersionValidator` is the kernel's validator for `StreamVersion`. The registry has no `tenantId` because every generation spans all tenants; the marker table is tenant data and leads with the tenant. Every refusal below is a plain `Error` with the message given. - validatorGenerationState: `const generationStateValidator = v.union(v.literal("building"), v.literal("verifying"), v.literal("verified"), v.literal("active"), v.literal("retired"), v.literal("aborted"), v.literal("purged"))` (E-8, D9) -- tableGenerations: `generations: defineTable({ readModel: v.string(), generation: v.number(), projectionVersion: v.number(), state: generationStateValidator, pauseRequired: v.boolean(), fence: v.number(), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), switchedAt: v.optional(v.number()), retiredAt: v.optional(v.number()), retireAfter: v.optional(v.number()) })` where `changedAt` and `changedBy` are the time and the operator of the last operator entry that changed the row, `switchedAt` is when the generation last became active, and `retiredAt` and `retireAfter` are set while it is `retired` and absent otherwise (E-8, E-48, D9, F13) +- tableGenerations: `generations: defineTable({ readModel: v.string(), generation: v.number(), projectionVersion: v.number(), state: generationStateValidator, pauseRequired: v.boolean(), fence: v.number(), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), switchedAt: v.optional(v.number()), retiredAt: v.optional(v.number()), retireAfter: v.optional(v.number()) })` where `changedAt` and `changedBy` are the time and the operator of the last operator entry that changed the row, which no batch writes, `switchedAt` is when the generation last became active, and `retiredAt` and `retireAfter` are set while it is `retired` and absent otherwise (E-8, E-48, D9, F13) - indexGenerationsByReadModel: `.index("by_read_model", ["readModel", "generation"])` (E-8) - indexGenerationsByReadModelUse: the next generation number at `startGeneration`, read as the last row in descending order, and the operator listing `getGenerations` (E-8, D9) - indexGenerationsByReadModelState: `.index("by_read_model_state", ["readModel", "state"])` (E-8) - indexGenerationsByReadModelStateUse: `generationsToWrite` in every command that maintains the read model, `activeGeneration` in every query over it, and the in-flight check of `startGeneration`, `switchGeneration` and `rollbackGeneration` (D9, E-8) - validatorBatchCursor: `const batchCursorValidator = v.object({ tenantId: v.string(), pageCursor: v.union(v.string(), v.null()), streamId: v.optional(v.string()), eventCursor: v.optional(v.union(v.string(), v.null())) })` the position of a pass, the tenant being iterated and the `paginator` cursor within it, `null` at the start of a tenant; `streamId` and `eventCursor` are set only by the batches of a history view (E-8, E-40, Law 11) - validatorProgressPass: `const progressPassValidator = v.union(v.literal("backfill"), v.literal("verify"), v.literal("purge"), v.literal("idle"))` naming the pass a batch chain is driving, `idle` when none is (E-40) -- tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), pass: progressPassValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position in the pass that `pass` names and `null` before that pass has read its first tenant; `batchesDone` counts every batch of the generation that reached its checkpoint, the batch that ends a pass included and a parked batch not, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted (E-40, D9, F13) +- tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), pass: progressPassValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position in the pass that `pass` names and `null` before that pass has read its first tenant; `batchesDone` counts every batch of the generation that reached its checkpoint, the batch that ends a pass included and a parked batch not, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted; every write of the row sets `updatedAt` to the writing mutation's `Date.now()` (E-40, D9, F13) - indexProgressByGeneration: `.index("by_generation", ["generationId"])` (E-40) - indexProgressByGenerationUse: the one read a batch and an operator entry make to find a generation's progress row; no command and no query over a read model reads the table (E-40, F1) - tableProjectionMarkers: `projectionMarkers: defineTable({ tenantId: v.string(), readModel: v.string(), generation: v.number(), entityKey: v.string(), aggregateKey: v.string(), contribution: v.number(), sourceVersions: v.array(streamVersionValidator) })` (E-8, D9, Law 11) diff --git a/design/specs/application/projection-contract.sdp.md b/design/specs/application/projection-contract.sdp.md index e3d2b24..6770c7c 100644 --- a/design/specs/application/projection-contract.sdp.md +++ b/design/specs/application/projection-contract.sdp.md @@ -61,7 +61,7 @@ A projection is the named, versioned, deterministic function that turns a contex ## Design -The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape with its `version` field, and `streamVersionValidator` is the kernel's validator; `D` is the context's DTO type; `Row` is the read-model document without the row conventions, which `applyProjection` adds. Validators use `convex/values`; the row conventions are a validator the read-model tables spread into their `defineTable`. +The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape with its `version` field, and `streamVersionValidator` is the kernel's validator; `D` is the context's DTO type; `Row` is the read-model document without the row conventions, which `applyProjection` adds. Validators use `convex/values`; the row conventions are a validator the read-model tables spread into their `defineTable`. In the write rule a row is older than the input when the entry of its `sourceVersions` whose `tenantId`, `contextId`, `streamType` and `streamId` equal the input's has a `version` below the input's; an equal version is not older, and a row that records no entry for the input's stream is older. - typeProjection: `type Projection> = { version: number; keyOf(tenantId: string, dto: D): string; project(tenantId: string, dto: D, versions: StreamVersion[]): Row | null }` (E-9, D8) - typeReadModel: `type ReadModel> = { name: string; table: string; rowBudgetBytes: number; projections: readonly [Projection, ...Projection[]] }` where `name` is the read model's key in the generation registry, `table` is the parent table that holds its rows, `rowBudgetBytes` is at most 65,536 with `defaultRowBudgetBytes` 16,384, and `projections` lists the versions the deployment can write, the latest first (E-9, D8, F13) @@ -103,4 +103,4 @@ The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape w - A reviewer confirms that no `project` function touches `ctx`, the database, the scheduler or the environment. - A reviewer confirms that every per-entity or cross-context read-model table spreads `rowConventions`, that every aggregate table spreads `aggregateRowConventions`, and that each carries the `by_key` index. - A reviewer confirms that `applyAggregate` compares the marker's `sourceVersions` before it touches `value`, that it reads no mode, and that a `null` contribution removes the marker and its recorded contribution. -- A reviewer confirms that a test writes one command into an active generation of one projection version and a generation being built of another and finds on each row its own generation's `projectionVersion` and that version's output, and that a test deletes a row through a `null` projection and finds the next live-updated write answering `inserted` in the active generation and in a generation in `verifying`, and `skipped-missing` in one in `building`. +- A reviewer confirms that a `convex-test` test on the fixture composition writes one `CreateTwiceSummarizedDocument` into an active generation of `documentTitle` at version 1 and a generation being built at version 2 and finds on each row its own generation's `projectionVersion`, that a backfill batch of each generation over a document whose `title` is the depot's `deletedTitle` leaves version 1's row with that title and no row at version 2, the two versions' one difference, and that a test deletes a row through a `null` projection, `applyProjection` in the backfill mode at version 2 over that document, and finds the next live-updated write of the same document at a higher stream version with a live title, `applyProjection` called with the mode `live-updated` because no fixture command amends a document under `documentTitle`, answering `inserted` in the active generation and in a generation in `verifying`, and `skipped-missing` in one in `building`. diff --git a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md index 4d8640f..91f7218 100644 --- a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md +++ b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md @@ -30,9 +30,9 @@ And writes {writes: "never stopped"} ## Verification — executable - Runs in the native tier; every test owns its disposable backend. -- The test seeds 350 orders across two tenants, 250 in the first and 100 in the second, with the order summary's generation 1 active, starts generation 2 with a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that `batchesDone` then stops advancing below 350 and that the cursor names a tenant. -- The test asserts that a command on an order whose row backfill had not yet written left no row in the building generation, and that a command on an order whose row existed updated it. -- After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that, once the generation is `verified`, every order enumerated by the Orders context in either tenant has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, that a batch scheduled under the fence the interrupt replaced wrote nothing, and that the backend's function log shows no context operation call and no action during the batches. -- The test asserts that no command of the run failed for reading the registry while the batches wrote their checkpoints, and that the generation rows changed only at the start, at the end of the backfill, at the end of the verify pass and at each operator entry. -- The test switches, asserts that the subscriptions on the list query in both tenants re-ran with the new generation and that generation 1 is `retired`, then runs one more `PlaceOrder` and deletes or cancels one earlier order so that its summary changes, rolls back by naming generation 1, asserts that the reopened generation reaches `verified` after one verify pass with a row for the new order and the changed order's current row before it becomes active, switches again, and asserts that generation 1 is active, generation 2 is `retired` and the subscriptions re-ran. -- A `convex-test` test of the same workflow runs each batch by calling it directly, so it interrupts after exactly 3 batches, rebuilds with a second projection version and asserts that each generation's rows carry their own generation's `projectionVersion` and that version's output under live writes, deletes a subject between the switch and the rollback and asserts that the reopened generation holds no row for it after its verify pass, and asserts every refusal of the operator entries by its message. +- The test, in a file whose own timeout is 300 s, installs the order summary's generation 1 through its first rebuild, seeds 350 orders across two tenants, 250 in the first in tenant-ID order and 100 in the second, starts generation 2 with `projectionVersion` 1 stated, a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that `batchesDone` then stops advancing below 350 and that the cursor names a tenant; while a chain is expected to run, a progress row whose `batchesDone` has not advanced for 30 s fails the test with the generation row and the progress row in the message. +- While the chain is interrupted the test sends `CancelOrder` to one of the second tenant's 100 seeded orders and to an order of the first tenant whose row generation 2 already holds, read from the table before the command; it asserts at once that the first tenant's order has its row in generation 2 `cancelled`, and, once generation 2 is `verified`, that the second tenant's order has its row in generation 2 `cancelled`, whichever pass wrote it. +- After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that, once the generation is `verified`, every order the Orders context's `list` enumerates in either tenant, read with admin access and `includeDeleted`, has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, that `backfillBatch` run with admin access under the fence the interrupt replaced leaves the progress row unchanged, that the function log shows no action and no completion during the batches other than a batch, an operator entry or a live command, and that the journals, read by operation, hold the event and receipt counts the live commands produced. +- The test asserts that no command of the run ends in a failure, that no completion record's `occInfo` names `generationProgress`, and that the completion records whose `occInfo` names `generations` number at most the operator entries and pass ends of the run times the callers the test keeps in flight plus one for the chain, and records those reruns with their `occInfo` as a measurement; it reads generation 2's row at the interrupt, after the resume and at `verified`, and asserts that the interrupt and the resume changed only `fence`, `changedAt` and `changedBy` and that from the resume to `verified` only `state` changed. +- The test switches, asserts from the function log that the list query ran again for each tenant's subscription after the switch and that each subscription's value holds the same rows, and that generation 1 is `retired`, then runs one more `PlaceOrder` and cancels one earlier order so that its summary changes, rolls back by naming generation 1, asserts that the reopened generation reaches `verified` after one verify pass with a row for the new order and the cancelled order's row `cancelled` before it becomes active, switches again, and asserts that generation 1 is active, generation 2 is `retired` and the list query ran again for each subscription. +- A `convex-test` test of the same workflow runs on the fixture composition with `documentTitle` and calls each batch directly, so it interrupts after exactly 3 batches; in one run generation 1 is installed at version 1 stated and generation 2 built at version 2 under live `CreateTwiceSummarizedDocument` writes, and it asserts that each generation's rows carry their own generation's `projectionVersion` and that a document whose `title` is the depot's `deletedTitle` has a row at version 1 and none at version 2; in a second run both generations are at version 2 and, after the switch, it deletes a document by `AmendDocument` with the depot's `deletedTitle`, rolls back by naming generation 1 and asserts that the reopened generation holds no row for it after its verify pass; and it asserts every refusal of the operator entries by its message. diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 5ea06ad..c24d40f 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -92,7 +92,7 @@ The registry, the progress row, the per-entity marker and the gate document are - From that commit every use case that maintains the read model writes the active generation, when there is one, and the one being built through `applyProjection`, in live-created mode for a stream it created and live-updated mode otherwise (D9, E-9) - Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building`, the pass is not `backfill` or the fence differs, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) - A batch that throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, and a developer error or a timeout ends the chain; the operator's `interruptGeneration` ends a chain on purpose, and `resumeGeneration` schedules a batch from the saved cursor with a new fence, at the batch size on the progress row; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) -- Each verify batch enumerates the sources again from the progress row's cursor, which the end of the backfill reset to the first tenant, and calls `applyProjection` in backfill mode for every DTO, so a missing row is inserted, an older one updated, and the row of a DTO that projects to `null` deleted; each such repair is counted as a miss; when the last tenant's last page is done the generation moves to `verified` and no batch is scheduled (D9, E-8, Sc L2-5) +- Each verify batch enumerates the sources again from the progress row's cursor, which the end of the backfill reset to `null`, the first tenant, and calls `applyProjection` in backfill mode for every DTO, so a missing row is inserted, an older one updated, and the row of a DTO that projects to `null` deleted; each such repair is counted as a miss; when the last tenant's last page is done the generation moves to `verified` and no batch is scheduled (D9, E-8, Sc L2-5) - `switchGeneration` sets the verified generation `active` and, when the read model has an active generation, sets that one `retired` with `retiredAt` and `retireAfter`, in one mutation; every query reads the active generation from the registry, so every subscription on the read model re-runs (D9, F7, F1) - [extension] During the rollback period the retired generation's rows are kept and not written; `rollbackGeneration` reopens the retired generation it is given as `verifying` with a new fence and a `null` cursor and schedules a verify batch, live commands write it again from that commit, its one verify pass repairs every row that is missing, older or left behind by a deleted subject, and the ordinary `switchGeneration` makes it active and retires the generation that served meanwhile (E-8, D9, Sc L2-5) - `abortGeneration` marks a building, verifying or verified generation `aborted`, leaves the active generation untouched, and resumes every closed gate entry that names this generation; an aborted rollback leaves the reopened generation `aborted`, never `retired` again (D9, Sc L2-6, E-8) @@ -104,7 +104,7 @@ The registry, the progress row, the per-entity marker and the gate document are ## Design -Each batch is one scheduled internal mutation in the parent. The rebuild schedules from a mutation, so a batch is scheduled only if its predecessor committed. The rebuild is an operator's work on parent tables; it is not deferred business work and creates no obligation. The library exports one plain helper per entry below, of the same name, taking `(ctx, config: RebuildConfig, args)` and returning the entry's value; a composition's `rebuild` module registers each entry over its helper with its own `internalMutation` and `internalQuery`, so the entries are reached as `internal.rebuild.startGeneration` and so on, and no public function starts or changes a generation. Every refusal is a plain `Error` with the message given; in a message, `name` is the read model's name, `n` the generation number and `state` the generation's state. +Each batch is one scheduled internal mutation in the parent. The rebuild schedules from a mutation, so a batch is scheduled only if its predecessor committed. The rebuild is an operator's work on parent tables; it is not deferred business work and creates no obligation. The library exports one plain helper per entry below, of the same name, taking `(ctx, config: RebuildConfig, args)` and returning the entry's value; a composition's `rebuild` module registers each entry over its helper with its own `internalMutation` and `internalQuery`, so the entries are reached as `internal.rebuild.startGeneration` and so on, and no public function starts or changes a generation. Every refusal is a plain `Error` with the message given; in a message, `name` is the read model's name, `n` the generation number and `state` the generation's state. Beside the helpers the library exports `batchSizeFor`, `rebuildBatchCeiling`, `limitPurgeBatch`, `resumeChain`, the constants `limitTenantFillBatch` and `limitGenerationsListed`, and the types `RebuildConfig`, `RebuildTarget`, `BatchRef` and `FillBatchRef`. Every patch of a progress row or of the `tenantFill` row sets `updatedAt` to the writing mutation's `Date.now()`; a field written as absent, `lastError`, `retiredAt` or `retireAfter`, is removed from the document, which a patch with `undefined` does; a batch never writes `changedAt` or `changedBy`, which only an operator entry sets, and when it ends a pass it patches the generation row's `state` and nothing else on that row. The enumeration is the context's `list` query with `includeDeleted`, the one read of context data per batch: one call returns a page of up to `batchSize` DTOs and a continuation cursor, built inside the component by `paginator` from `convex-helpers` over the `by_identity` index that includes deleted subjects, and capped by the query at its byte-derived page bound. A batch reads `page`, `isDone` and `continueCursor` and ignores any split status, because the rows up to `continueCursor` are complete. A DTO carries the stream version the projection compares, as the queries contract's `toDto(state, meta)` maps it onto `version`. Deleted subjects reach the projection as DTOs the projection maps to `null`, so their rows are deleted rather than left behind. @@ -112,7 +112,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - convexSurface: `startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration`, `purgeGeneration` and `fillTenants` as the operator entries, `backfillBatch`, `verifyBatch`, `purgeBatch` and `fillTenantsBatch` as the scheduled batches, all internal mutations, and `getGenerations` as an internal query, in the composition's `rebuild` module; the gate entries are in the write-pause contract (D9, E-8, E-48) - typeRebuildTarget: `type RebuildTarget = { readModel: ReadModel; source: SourceRef; list: FunctionReference<"query", "internal", ListArgs, PaginationResult> }` one per-entity read model the deployment can rebuild, its one source stream type, and the reference to that stream type's `list` query in its context component, such as `components.orders.queries.order.list` (E-8, E-9, D10) - typeRebuildConfig: `type RebuildConfig = { targets: readonly RebuildTarget[]; refs: { backfillBatch: BatchRef; verifyBatch: BatchRef; purgeBatch: BatchRef; fillTenantsBatch: FillBatchRef } }` where `type BatchRef = FunctionReference<"mutation", "internal", { generationId: Id<"generations">; fence: number }, null>`, `type FillBatchRef = FunctionReference<"mutation", "internal", { fence: number }, null>` and `refs` are the composition's own registered batches, which a batch schedules by reference (E-40, E-49, F8) -- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; the tenant list check, throwing `"The tenant list is empty while grants exist; fillTenants fills it from the grants table"` when `nextTenant(ctx, null)` answers `null` and `ctx.db.query("grants").first()` answers a row, and `"The tenant list is being filled; startGeneration waits until fillTenants has ended"` when the `tenantFill` row is in the pass `fill`; then it inserts the generation row with the next number, `state: "building"`, `pauseRequired: false`, since a `RebuildTarget` is a per-entity read model and the paused start of a history view is given only in outline, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `pass: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48, E-49) +- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; the tenant list check, throwing `"The tenant list is empty while grants exist; fillTenants fills it from the grants table"` when `nextTenant(ctx, null)` answers `null` and `ctx.db.query("grants").first()` answers a row, and `"The tenant list is being filled; startGeneration waits until fillTenants has ended"` when the `tenantFill` row is in the pass `fill`; then it inserts the generation row with the next number, one above the last row of `by_read_model` in descending order and 1 when there is none, `state: "building"`, `pauseRequired: false`, since a `RebuildTarget` is a per-entity read model and the paused start of a history view is given only in outline, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `pass: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48, E-49) - fnBatchSizeFor: `batchSizeFor(readModel: ReadModel, requested?: number): number` throwing `"The batch size must be a whole number of at least 1, and " + requested + " is not"` unless `requested` is absent or an integer of at least 1, and returning `min(requested ?? rebuildBatchCeiling(readModel), rebuildBatchCeiling(readModel))`, so a batch size of 0, a negative one and a fraction are refused and a larger one is lowered to the ceiling (E-40, F13) - fnBackfillBatch: `export const backfillBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9, E-40) - fnVerifyBatch: `export const verifyBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9) @@ -122,33 +122,33 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - step2: it takes its tenant: the cursor's `tenantId`, or `nextTenant(ctx, null)` when the cursor is `null`; when the tenant list is empty the batch has no tenant, reads the gate as step 3 states, and goes to step 6 with the pass done (E-49, Law 11) - step3: it calls the write pause's `gateAllows(ctx, ["all", "tenant:" + tenantId], generationId)`, with `["all"]` alone when it has no tenant, and when the answer is not allowed it parks: it patches the progress row's `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt`, schedules nothing and returns, so the generation waits for `resumeGeneration` after the gate reopens; no batch calls `assertWritable`, because a batch parks its generation rather than failing its scheduled mutation with a thrown refusal (E-8, E-42, D9) - step4: it calls `list` once as `fnEnumerateCall` states, with the cursor's `pageCursor` (D10, E-40) -- step5: for each DTO of the page it calls `applyProjection(ctx, target.readModel, { tenantId, dto, versions: [dto.version], mode: "backfill", generations: [thisGeneration] })` for this generation alone, adding one to `rowsWritten` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` for `skipped-newer` or `unchanged` (D9, E-9) -- step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) -- verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `pass` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and the progress row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) -- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(Math.min(batchSize, limitPurgeBatch))` with `batchSize` the progress row's, deletes each row and adds their number to `rowsPurged`; when it deleted that many rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`; a pass that is not done checkpoints and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) +- step5: for each DTO of the page it calls `applyProjection(ctx, target.readModel, { tenantId, dto, versions: [dto.version], mode: "backfill", generations: [thisGeneration] })` for this generation alone, where `thisGeneration` is `{ role, generation: n, projectionVersion }` with the role `typeWritableGeneration` of `spec:application.generation-registry` gives the generation's state, adding one to `rowsWritten` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` for `skipped-newer` or `unchanged` (D9, E-9) +- step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"` and nothing else on it, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) +- verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `pass` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) +- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(Math.min(batchSize, limitPurgeBatch))` with `batchSize` the progress row's, deletes each row and adds their number to `rowsPurged`; when it deleted that many rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`, a purge cursor being `{ tenantId, pageCursor: null }`; a pass that is not done checkpoints, patching the progress row with the cursor, `rowsPurged`, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) - purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) -- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) -- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, clearing `lastError` and leaving the progress row's `batchSize` as it is, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) +- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator` and `updatedAt`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) +- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to no `lastError` and `updatedAt` and leaving its `batchSize`, cursor and counts as they are, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) - fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain`; it takes no batch size, so a resumed pass runs at the progress row's `batchSize`, a purge capped by `limitPurgeBatch`; for a generation in any state whose progress row names a pass, so it continues a backfill or verify pass of a generation in `building` or `verifying` and a started purge of a `retired` or `aborted` one, which is how an operator continues a purge a restore stranded (D9, Sc L2-5, E-40, E-48) - fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) -- fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `pass` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, the in-flight error of `startGeneration` when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0` and no `lastError`, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) -- fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and `"Generation " + n + " of " + name + " is " + state + "; abortGeneration needs building, verifying or verified"` unless the state is one of the three; otherwise patching the row to `state: "aborted"`, fence plus 1, `changedAt` and `changedBy`, patching the progress row to `pass: "idle"` and `lastError: "aborted: " + reason`, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8, E-48) +- fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `pass` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, the in-flight error of `startGeneration`, naming that generation's number and state, when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, no `lastError` and `updatedAt`, with `batchSize` and the other counts kept, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) +- fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and `"Generation " + n + " of " + name + " is " + state + "; abortGeneration needs building, verifying or verified"` unless the state is one of the three; otherwise patching the row to `state: "aborted"`, fence plus 1, `changedAt` and `changedBy`, patching the progress row to `pass: "idle"`, `lastError: "aborted: " + reason` and `updatedAt`, with the cursor and the counts kept, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8, E-48) - fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged; resumeGeneration continues it"` when `pass` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `pass: "purge"`, `cursor: null` and no `lastError`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) - fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(v.object({ generation: generationDoc, progress: progressDoc })), handler })` the newest `limitGenerationsListed` generations of the read model by `by_read_model` in descending order, each with its progress row, where `generationDoc` and `progressDoc` are the two tables' document validators with `_id` and `_creationTime`; an operator entry that only reads, so it takes no operator (E-8, E-40, E-48, D19) - tableTenantFill: `tenantFill: defineTable({ pass: v.union(v.literal("fill"), v.literal("idle")), fence: v.number(), cursor: v.union(v.string(), v.null()), batchesDone: v.number(), tenantsRead: v.number(), tenantsInserted: v.number(), lastError: v.optional(v.string()), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), updatedAt: v.number() })` in `readModelTables`, at most one row per deployment, the progress row of the tenant fill and nothing else's; operational data of the deployment with no tenant dimension and no index, read with `ctx.db.query("tenantFill").first()`; `cursor` is the last tenant ID the walk passed, `null` before its first, `tenantsRead` counts the tenants the walk found and `tenantsInserted` the rows it inserted (E-49, E-40, E-48) - fnFillTenants: `export const fillTenants = internalMutation({ args: { operator: v.string() }, returns: v.null(), handler })` in order: `assertOperator`; the restore door, throwing `"fillTenants is refused while the restore door is closed"` when `process.env.MAINTENANCE_MODE` is exactly `restore`, because the fill writes the tenant list that a restore replaces, and no other read of the gate, because the fill writes no tenant's data and so no closed tenant scope stops it; for each target of `config.targets` the in-flight check by `by_read_model_state`, throwing `"Read model " + name + " has generation " + n + " in " + state + "; fillTenants needs no generation in flight"`; then, with `now` the mutation's `Date.now()`, when no `tenantFill` row exists it inserts one with `pass: "fill"`, `fence: 1`, `cursor: null`, every count 0, `startedAt`, `changedAt` and `updatedAt` at `now` and `startedBy` and `changedBy` the operator, and when one exists it patches it to `pass: "fill"`, fence plus 1, the cursor and the counts kept when the pass was `fill` and `null` and 0 when it was `idle`, no `lastError`, `changedAt`, `changedBy` and `updatedAt`; then it schedules `refs.fillTenantsBatch` with the row's fence through `ctx.scheduler.runAfter(0, ...)`; so a call continues a fill whose chain a restore, an error or a closed gate ended, and a call after a finished fill walks the grants again and inserts only what is missing (E-49, E-48, E-40, F8) - fnFillTenantsBatch: `export const fillTenantsBatch = internalMutation({ args: { fence: v.number() }, returns: v.null(), handler })` (E-49, E-40) - fillSteps: `fillTenantsBatch` loads the `tenantFill` row and returns without writing when it is absent, `pass` is not `fill` or `fence` differs from its argument; it calls `gateAllows(ctx, ["all"])` and, when the answer is not allowed, parks as step 3 does, patching `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt` and scheduling nothing, so the fill waits for `fillTenants` after the gate reopens; otherwise it repeats at most `limitTenantFillBatch` times: it reads `ctx.db.query("grants").withIndex("by_principal", (q) => (cursor === null ? q : q.gt("tenantId", cursor))).first()`, stops when that answers no row, and otherwise reads `tenants` by `by_tenant` for the grant's `tenantId`, inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, adds one to `tenantsRead` and, for an insert, one to `tenantsInserted`, and sets the cursor to that `tenantId`; a batch that took `limitTenantFillBatch` tenants patches the row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt` and schedules `refs.fillTenantsBatch` with the same fence; a batch that found no further grant patches the row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt` and schedules nothing (E-49, E-40, F8, Law 11) -- limitTenantFillBatch: 8 tenants per fill batch; a grant row's bytes are bounded only by the 1 MiB document limit, so a batch reads at most 8 grant rows, 8 MiB, one per tenant through `by_principal`, at most 8 tenant rows of at most `limitIdLength` bytes of tenant ID each, the gate document and the `tenantFill` row, and writes at most 8 tenant rows and the `tenantFill` row, under the 16 MiB read and write ceilings (E-49, E-40, F13) +- limitTenantFillBatch: 8 tenants per fill batch, a library constant; a grant row's bytes are bounded only by the 1 MiB document limit, so a batch reads at most 8 grant rows, 8 MiB, one per tenant through `by_principal`, at most 8 tenant rows of at most `limitIdLength` bytes of tenant ID each, the gate document and the `tenantFill` row, and writes at most 8 tenant rows and the `tenantFill` row, under the 16 MiB read and write ceilings (E-49, E-40, F13) - errorUnknownGeneration: every entry and every helper that takes a `generationId` throws `"No generation " + generationId` when the row is absent, and a batch returns without writing instead (E-8) - stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its pass is not `idle`; `resumeGeneration` takes no batch size, so the operator answers a stalled backfill or verify pass with `abortGeneration` and a new `startGeneration` with a smaller `batchSize`, down to 1, and a purge batch deletes rows of a generation nothing else writes (E-40, F1, F9, Probe 6) - occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch; a batch reads the generation row and never writes it outside the end of a pass, and it writes the progress row, which no command reads, so a batch's checkpoint re-runs no command (F1, D9, Probe 6, E-40) - limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) -- limitPurgeBatch: `min(500, floor(4194304 / rowBudgetBytes))` rows, the cap on the rows a purge batch deletes, which deletes the progress row's `batchSize` when that is smaller; 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) +- limitPurgeBatch: `limitPurgeBatch(readModel: ReadModel): number`, `min(500, floor(4194304 / readModel.rowBudgetBytes))` rows, the cap on the rows a purge batch deletes, which deletes the progress row's `batchSize` when that is smaller; 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) - limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch returns without writing (E-40, F13) - limitBuildingGenerations: at most one generation in `building`, `verifying` or `verified` per read model at a time, so a command writes at most two generations, the active one and that one (E-8, E-9) - limitRollbackPeriod: `[extension]` 7 days by default, stored as `retireAfter` on the retired row; the retired generation is kept and not written, and a rollback at or after `retireAfter` is refused (E-8, D9) -- limitGenerationsListed: 20 generations per `getGenerations` answer, the newest first (E-8, F13) +- limitGenerationsListed: 20 generations per `getGenerations` answer, the newest first, a library constant (E-8, F13) - fnHistoryPageCall: `ctx.runQuery(components.orders.queries.order.history, { tenantId, streamId, paginationOpts: { numItems: limitHistoryBatch, cursor: eventCursor } })` the one read of context data a history batch makes beside the `list` call of `numItems` 1 that gives it its next stream, returning a `paginator` page of envelopes in version order that the batch folds through `applyHistoryProjection` (D2, D10, E-9, E-24) - historyStep3: for a history view a batch replaces steps 4 and 5: when the cursor names no stream it calls `list` with `numItems` 1 from the page cursor to take the next stream and sets `eventCursor` to `null`; it then calls `history` for that stream with `limitHistoryBatch` events from `eventCursor`, calls `applyHistoryProjection` in backfill mode for the building generation with the page, and saves `eventCursor`, or clears `streamId` and advances the page cursor when the page reports `isDone` (D9, E-9, E-24, E-40) - historyVerify: a verify batch of a history view enumerates the sources with `list` and, per stream, reads its last event through `history` with `fromVersion` at the stream's version and `numItems` 1, finds that event's row by `keyOf`, and counts a miss when the row is absent or its `sourceVersions` record a lower version for the stream; a miss is repaired by folding the stream's history in place through the same history batches from `eventCursor` null (D9, E-9, Sc L2-6) diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 72ed4bd..84f6bb7 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -87,7 +87,7 @@ The harness is plain test support with no Convex function in it. The entries who - productionIsolation: [extension] the lint rule `production/no-test-code` fails any module under `example/` whose relative import resolves into `fixture/`, `harness/` or `tests/`, and it checks relative imports only, and nothing that injects a fault and no fixture function that stands for trusted server code, such as `nonUiCaller:send`, is registered in the production composition (E-15, Acceptance scenarios) - fixtureComponent: [extension] the fixture composition mounts one plain component as `annex`, the fixture context `depot` under its `contextId` and a second fixture context `yard` under its own; every other fixture context gets its own name (E-15) - fixtureSecondContext: [extension] a fixture command calls `depot` and then `yard`, whose operation throws a plain error after its own state write for the fault title its stream registration refuses, as the depot's faults are, so a technical failure in a use case's second context runs on the fixture composition (E-15, Sc L2-1) -- fixtureReadModel: [extension] the fixture composition declares the fixture read model `documentSummary`, and a fixture command of its own binds it and, after it, a second fixture read model, `documentTitle`, whose projection throws a plain error for its fault title, so a failure after step 9's first row write runs on the fixture composition and shows that step 9's writes roll back with the command (E-15, Sc L1-2, D8) +- fixtureReadModel: [extension] the fixture composition declares the fixture read model `documentSummary`, and a fixture command of its own binds it and, after it, a second fixture read model, `documentTitle`, whose projections throw a plain error for its fault title, so a failure after step 9's first row write runs on the fixture composition and shows that step 9's writes roll back with the command; `documentTitle` declares `projections` of version 2 and version 1, the latest first, where version 1 projects every document to `{ documentId, title }` and version 2 projects the document whose `title` is the depot's `deletedTitle`, the deleted document, to `null` and every other document as version 1 does, both throwing for the fault title before anything else, so a generation of either version holds the rows the fault scenario reads; every setup that installs `documentTitle` states `projectionVersion` 1 to `startGeneration`, so a generation at version 2 is a test's own step, and `documentSummary` declares version 1 alone (E-15, Sc L1-2, D8, E-9) - fixtureTrustedCaller: [extension] the fixture action `nonUiCaller:send`, which an ordinary client calls, runs a fixture command's internal entry through `ctx.runMutation` with the namespace and actor kind the call names and the caller's own identity as the actor; it stands for the trusted server code that calls an internal entry, because no caller under test carries the admin key, and it exists in the fixture composition only, never in the production composition (E-15, E-13, D11) - watchMode: [extension] `npm run dev` starts one backend through the harness and runs `convex dev` against it as a long-lived child, so that a composition's generated files follow its files as they are saved, with the composition chosen by an argument and, with no argument, the fixture composition; it is a developer's tool and no test runs through it; it is the one place outside `AdminAccess` that uses the admin key, for that CLI call and to read the backend's site URL, and it prints neither (E-15) - sweepOwnership: [extension] the sweep reads a backend's ownership record, then the process table, then signals by process id; a process id cannot change hands while its process is alive, so the interval between the read and the signal matters only for a backend that exits during the sweep, and the harness does not close it: no call available to a process that did not spawn the backend does; a process whose command line cannot be read, such as a kernel thread that took a dead backend's process id, keeps that backend's directory and fails that sweep, and the next run starts from a fresh root (E-15) From 91457a40e6e94850ca656a77da9fdbabaed8cfc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 03:59:29 +0200 Subject: [PATCH 077/155] build: 3b online read-model rebuild --- example/convex/_generated/api.d.ts | 2 + example/convex/orderSummary.ts | 24 +- example/convex/ordering.ts | 2 +- example/convex/readModels.ts | 21 +- example/convex/rebuild.ts | 114 +++++ fixture/convex/_generated/api.d.ts | 2 + fixture/convex/documentTitles.ts | 35 +- fixture/convex/migrations.ts | 17 +- fixture/convex/readModels.ts | 22 +- fixture/convex/rebuild.ts | 120 +++++ fixture/convex/summaries.ts | 20 +- fixture/convex/summarizedTwice.ts | 2 +- src/read-model/generations.ts | 51 +- src/read-model/index.ts | 37 +- src/read-model/projection.ts | 55 ++- src/read-model/rebuild.ts | 747 +++++++++++++++++++++++++++++ src/read-model/tables.ts | 57 ++- 17 files changed, 1183 insertions(+), 145 deletions(-) create mode 100644 example/convex/rebuild.ts create mode 100644 fixture/convex/rebuild.ts create mode 100644 src/read-model/rebuild.ts diff --git a/example/convex/_generated/api.d.ts b/example/convex/_generated/api.d.ts index 7df9760..4a7dc62 100644 --- a/example/convex/_generated/api.d.ts +++ b/example/convex/_generated/api.d.ts @@ -14,6 +14,7 @@ import type * as orderQueries from "../orderQueries.js"; import type * as orderSummary from "../orderSummary.js"; import type * as ordering from "../ordering.js"; import type * as readModels from "../readModels.js"; +import type * as rebuild from "../rebuild.js"; import type * as receiving from "../receiving.js"; import type { @@ -29,6 +30,7 @@ declare const fullApi: ApiFromModules<{ orderSummary: typeof orderSummary; ordering: typeof ordering; readModels: typeof readModels; + rebuild: typeof rebuild; receiving: typeof receiving; }>; diff --git a/example/convex/orderSummary.ts b/example/convex/orderSummary.ts index 9ba1f1c..dae6019 100644 --- a/example/convex/orderSummary.ts +++ b/example/convex/orderSummary.ts @@ -28,15 +28,17 @@ export const orderSummary: ReadModel = { name: "orderSummary", table: "orderSummaries", rowBudgetBytes: defaultRowBudgetBytes, - projection: { - version: 1, - keyOf: (_tenantId, order) => order.orderId, - project: (_tenantId, { orderId, status, lines, total, placedAt }) => ({ - orderId, - status, - lineCount: lines.length, - total, - placedAt, - }), - }, + projections: [ + { + version: 1, + keyOf: (_tenantId, order) => order.orderId, + project: (_tenantId, { orderId, status, lines, total, placedAt }) => ({ + orderId, + status, + lineCount: lines.length, + total, + placedAt, + }), + }, + ], }; diff --git a/example/convex/ordering.ts b/example/convex/ordering.ts index 92ff9ba..f7c419d 100644 --- a/example/convex/ordering.ts +++ b/example/convex/ordering.ts @@ -19,7 +19,7 @@ export const placeOrderPermission = "orders.place"; export const cancelOrderPermission = "orders.cancel"; // The largest order PlaceOrder accepts: the example's promise, chosen under the adapter's ceilings. export const maxOrderLines = 100; -const orderSource = { contextId: "orders", streamType: "order" }; +export const orderSource = { contextId: "orders", streamType: "order" }; const stockItemSource = { contextId: "inventory", streamType: "stockItem" }; const placeOrderInput = v.object({ orderId: v.string(), diff --git a/example/convex/readModels.ts b/example/convex/readModels.ts index c5a1cb6..8a23594 100644 --- a/example/convex/readModels.ts +++ b/example/convex/readModels.ts @@ -1,41 +1,26 @@ -// The production composition's read-model functions: the first activation an operator runs with admin -// access, and the parent list over the order summary in its active generation. +// The parent list over the read model in its active generation. import { paginationOptsValidator, paginationResultValidator, } from "convex/server"; import { v } from "convex/values"; -import { actorValidator, authorizeQuery } from "../../src/command/index.js"; +import { authorizeQuery } from "../../src/command/index.js"; import { boundedPage, streamVersionValidator, } from "../../src/context/index.js"; import { - activateFirstGeneration, activeGeneration, limitReadModelList, readModelView, - type AnyReadModel, } from "../../src/read-model/index.js"; -import { internalMutation, query } from "./_generated/server.js"; +import { query } from "./_generated/server.js"; import { orderSummary, orderSummaryFields, orderSummaryStatus, } from "./orderSummary.js"; export const readOrdersPermission = "orders.read"; -const readModels: readonly AnyReadModel[] = [orderSummary]; -// Run once for each read model, before the first command that writes it. -export const activate = internalMutation({ - args: { readModel: v.string(), startedBy: actorValidator }, - returns: v.number(), - handler: async (ctx, { readModel, startedBy }) => { - const declared = readModels.find(({ name }) => name === readModel); - if (declared === undefined) - throw new Error(`This deployment declares no read model ${readModel}`); - return activateFirstGeneration(ctx, declared, startedBy); - }, -}); const orderSummaryView = v.object({ tenantId: v.string(), key: v.string(), diff --git a/example/convex/rebuild.ts b/example/convex/rebuild.ts new file mode 100644 index 0000000..4f43e51 --- /dev/null +++ b/example/convex/rebuild.ts @@ -0,0 +1,114 @@ +// The composition's thirteen internal read-model rebuild functions. +import { v } from "convex/values"; +import * as rebuild from "../../src/read-model/rebuild.js"; +import { readModelTables } from "../../src/read-model/tables.js"; +import { components, internal } from "./_generated/api.js"; +import { internalMutation, internalQuery } from "./_generated/server.js"; +import { orderSummary } from "./orderSummary.js"; +import { orderSource } from "./ordering.js"; +const config: rebuild.RebuildConfig = { + targets: [ + { + readModel: orderSummary, + source: orderSource, + list: components.orders.queries.order.list, + }, + ], + refs: { + backfillBatch: internal.rebuild.backfillBatch, + verifyBatch: internal.rebuild.verifyBatch, + purgeBatch: internal.rebuild.purgeBatch, + fillTenantsBatch: internal.rebuild.fillTenantsBatch, + }, +}; +const batchArgs = { generationId: v.id("generations"), fence: v.number() }; +const generationDoc = v.object({ + ...readModelTables.generations.validator.fields, + _id: v.id("generations"), + _creationTime: v.number(), +}); +const progressDoc = v.object({ + ...readModelTables.generationProgress.validator.fields, + _id: v.id("generationProgress"), + _creationTime: v.number(), +}); +export const startGeneration = internalMutation({ + args: { + readModel: v.string(), + projectionVersion: v.optional(v.number()), + batchSize: v.optional(v.number()), + operator: v.string(), + }, + returns: v.id("generations"), + handler: (ctx, args) => rebuild.startGeneration(ctx, config, args), +}); +export const interruptGeneration = internalMutation({ + args: { generationId: v.id("generations"), operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.interruptGeneration(ctx, config, args), +}); +export const resumeGeneration = internalMutation({ + args: { generationId: v.id("generations"), operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.resumeGeneration(ctx, config, args), +}); +export const switchGeneration = internalMutation({ + args: { + generationId: v.id("generations"), + rollbackPeriodMs: v.optional(v.number()), + operator: v.string(), + }, + returns: v.null(), + handler: (ctx, args) => rebuild.switchGeneration(ctx, config, args), +}); +export const rollbackGeneration = internalMutation({ + args: { generationId: v.id("generations"), operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.rollbackGeneration(ctx, config, args), +}); +export const abortGeneration = internalMutation({ + args: { + generationId: v.id("generations"), + reason: v.string(), + operator: v.string(), + }, + returns: v.null(), + handler: (ctx, args) => rebuild.abortGeneration(ctx, config, args), +}); +export const purgeGeneration = internalMutation({ + args: { generationId: v.id("generations"), operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.purgeGeneration(ctx, config, args), +}); +export const fillTenants = internalMutation({ + args: { operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.fillTenants(ctx, config, args), +}); +export const backfillBatch = internalMutation({ + args: batchArgs, + returns: v.null(), + handler: (ctx, args) => rebuild.backfillBatch(ctx, config, args), +}); +export const verifyBatch = internalMutation({ + args: batchArgs, + returns: v.null(), + handler: (ctx, args) => rebuild.verifyBatch(ctx, config, args), +}); +export const purgeBatch = internalMutation({ + args: batchArgs, + returns: v.null(), + handler: (ctx, args) => rebuild.purgeBatch(ctx, config, args), +}); +export const fillTenantsBatch = internalMutation({ + args: { fence: v.number() }, + returns: v.null(), + handler: (ctx, args) => rebuild.fillTenantsBatch(ctx, config, args), +}); +export const getGenerations = internalQuery({ + args: { readModel: v.string() }, + returns: v.array( + v.object({ generation: generationDoc, progress: progressDoc }), + ), + handler: (ctx, args) => rebuild.getGenerations(ctx, config, args), +}); diff --git a/fixture/convex/_generated/api.d.ts b/fixture/convex/_generated/api.d.ts index fca6fa8..66614f2 100644 --- a/fixture/convex/_generated/api.d.ts +++ b/fixture/convex/_generated/api.d.ts @@ -29,6 +29,7 @@ import type * as orders from "../orders.js"; import type * as parentList from "../parentList.js"; import type * as readCost from "../readCost.js"; import type * as readModels from "../readModels.js"; +import type * as rebuild from "../rebuild.js"; import type * as rejectionCommands from "../rejectionCommands.js"; import type * as scheduledRows from "../scheduledRows.js"; import type * as summaries from "../summaries.js"; @@ -65,6 +66,7 @@ declare const fullApi: ApiFromModules<{ parentList: typeof parentList; readCost: typeof readCost; readModels: typeof readModels; + rebuild: typeof rebuild; rejectionCommands: typeof rejectionCommands; scheduledRows: typeof scheduledRows; summaries: typeof summaries; diff --git a/fixture/convex/documentTitles.ts b/fixture/convex/documentTitles.ts index 5e39556..f332d23 100644 --- a/fixture/convex/documentTitles.ts +++ b/fixture/convex/documentTitles.ts @@ -7,6 +7,7 @@ import { defaultRowBudgetBytes, type ReadModel, } from "../../src/read-model/index.js"; +import { deletedTitle } from "./depot/streams.js"; // A document created under this title makes the title projection throw. export const projectionFaultTitle = "fault: in the documentTitle projection"; // The projected fields, which the table spreads. @@ -25,15 +26,29 @@ export const documentTitle: ReadModel = { name: "documentTitle", table: "documentTitles", rowBudgetBytes: defaultRowBudgetBytes, - projection: { - version: 1, - keyOf: (_tenantId, document) => document.documentId, - project: (_tenantId, { documentId, title }) => { - if (title === projectionFaultTitle) - throw new Error( - `Fault injected: ${projectionFaultTitle}, projecting document ${documentId}`, - ); - return { documentId, title }; + projections: [ + { + version: 2, + keyOf: (_tenantId, document) => document.documentId, + project: (_tenantId, { documentId, title }) => { + if (title === projectionFaultTitle) + throw new Error( + `Fault injected: ${projectionFaultTitle}, projecting document ${documentId}`, + ); + if (title === deletedTitle) return null; + return { documentId, title }; + }, }, - }, + { + version: 1, + keyOf: (_tenantId, document) => document.documentId, + project: (_tenantId, { documentId, title }) => { + if (title === projectionFaultTitle) + throw new Error( + `Fault injected: ${projectionFaultTitle}, projecting document ${documentId}`, + ); + return { documentId, title }; + }, + }, + ], }; diff --git a/fixture/convex/migrations.ts b/fixture/convex/migrations.ts index 3a50e91..c641585 100644 --- a/fixture/convex/migrations.ts +++ b/fixture/convex/migrations.ts @@ -36,7 +36,9 @@ const summaryMigration = makeFunctionReference( const enumerationMigration = makeFunctionReference( "migrations:enumeration", ) as unknown as MigrationFunctionReference; -type Document = Parameters[1]; +type Document = Parameters< + (typeof documentSummary.projections)[0]["project"] +>[1]; async function writeSummary(ctx: MutationCtx, tenantId: string, dto: Document) { const existing = await ctx.db @@ -48,7 +50,7 @@ async function writeSummary(ctx: MutationCtx, tenantId: string, dto: Document) { if (existing && existing.sourceVersions[0]!.version >= dto.version.version) return; const row = { - ...documentSummary.projection.project(tenantId, dto, [dto.version])!, + ...documentSummary.projections[0].project(tenantId, dto, [dto.version])!, tenantId, generation: 2, key: dto.documentId, @@ -242,10 +244,15 @@ export const contextBatch = internalMutation({ version: dto.version.version, }); } - await ctx.db.patch("generations", generation._id, { + const progress = (await ctx.db + .query("generationProgress") + .withIndex("by_generation", (q) => q.eq("generationId", generation._id)) + .unique())!; + await ctx.db.patch("generationProgress", progress._id, { cursor: { tenantId: "tenant", pageCursor: page.continueCursor }, - batchesDone: generation.batchesDone + 1, - rowsWritten: generation.rowsWritten + page.page.length, + batchesDone: progress.batchesDone + 1, + rowsWritten: progress.rowsWritten + page.page.length, + updatedAt: Date.now(), }); if ( page.page.some( diff --git a/fixture/convex/readModels.ts b/fixture/convex/readModels.ts index eeb4cf5..cfee8ca 100644 --- a/fixture/convex/readModels.ts +++ b/fixture/convex/readModels.ts @@ -1,40 +1,24 @@ -// The fixture composition's read-model functions: the first activation an operator runs with admin -// access, and the parent list over the document summary in its active generation. +// The parent list over the read model in its active generation. import { paginationOptsValidator, paginationResultValidator, } from "convex/server"; import { v } from "convex/values"; -import { actorValidator, authorizeQuery } from "../../src/command/index.js"; +import { authorizeQuery } from "../../src/command/index.js"; import { boundedPage } from "../../src/context/index.js"; import { streamVersionValidator } from "../../src/context/index.js"; import { - activateFirstGeneration, activeGeneration, limitReadModelList, readModelView, - type AnyReadModel, } from "../../src/read-model/index.js"; -import { internalMutation, query } from "./_generated/server.js"; -import { documentTitle } from "./documentTitles.js"; +import { query } from "./_generated/server.js"; import { documentSummary, documentSummaryFields, documentSummaryStatus, } from "./summaries.js"; export const readPermission = "depot.read"; -const readModels: readonly AnyReadModel[] = [documentSummary, documentTitle]; -// Run once for each read model, before the first command that writes it. -export const activate = internalMutation({ - args: { readModel: v.string(), startedBy: actorValidator }, - returns: v.number(), - handler: async (ctx, { readModel, startedBy }) => { - const declared = readModels.find(({ name }) => name === readModel); - if (declared === undefined) - throw new Error(`This deployment declares no read model ${readModel}`); - return activateFirstGeneration(ctx, declared, startedBy); - }, -}); const documentSummaryView = v.object({ tenantId: v.string(), key: v.string(), diff --git a/fixture/convex/rebuild.ts b/fixture/convex/rebuild.ts new file mode 100644 index 0000000..ca9e1ef --- /dev/null +++ b/fixture/convex/rebuild.ts @@ -0,0 +1,120 @@ +// The composition's thirteen internal read-model rebuild functions. +import { v } from "convex/values"; +import * as rebuild from "../../src/read-model/rebuild.js"; +import { readModelTables } from "../../src/read-model/tables.js"; +import { components, internal } from "./_generated/api.js"; +import { internalMutation, internalQuery } from "./_generated/server.js"; +import { documentSummary } from "./summaries.js"; +import { documentTitle } from "./documentTitles.js"; +import { documentSource } from "./summarizedTwice.js"; +const config: rebuild.RebuildConfig = { + targets: [ + { + readModel: documentSummary, + source: documentSource, + list: components.depot.queries.document.list, + }, + { + readModel: documentTitle, + source: documentSource, + list: components.depot.queries.document.list, + }, + ], + refs: { + backfillBatch: internal.rebuild.backfillBatch, + verifyBatch: internal.rebuild.verifyBatch, + purgeBatch: internal.rebuild.purgeBatch, + fillTenantsBatch: internal.rebuild.fillTenantsBatch, + }, +}; +const batchArgs = { generationId: v.id("generations"), fence: v.number() }; +const generationDoc = v.object({ + ...readModelTables.generations.validator.fields, + _id: v.id("generations"), + _creationTime: v.number(), +}); +const progressDoc = v.object({ + ...readModelTables.generationProgress.validator.fields, + _id: v.id("generationProgress"), + _creationTime: v.number(), +}); +export const startGeneration = internalMutation({ + args: { + readModel: v.string(), + projectionVersion: v.optional(v.number()), + batchSize: v.optional(v.number()), + operator: v.string(), + }, + returns: v.id("generations"), + handler: (ctx, args) => rebuild.startGeneration(ctx, config, args), +}); +export const interruptGeneration = internalMutation({ + args: { generationId: v.id("generations"), operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.interruptGeneration(ctx, config, args), +}); +export const resumeGeneration = internalMutation({ + args: { generationId: v.id("generations"), operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.resumeGeneration(ctx, config, args), +}); +export const switchGeneration = internalMutation({ + args: { + generationId: v.id("generations"), + rollbackPeriodMs: v.optional(v.number()), + operator: v.string(), + }, + returns: v.null(), + handler: (ctx, args) => rebuild.switchGeneration(ctx, config, args), +}); +export const rollbackGeneration = internalMutation({ + args: { generationId: v.id("generations"), operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.rollbackGeneration(ctx, config, args), +}); +export const abortGeneration = internalMutation({ + args: { + generationId: v.id("generations"), + reason: v.string(), + operator: v.string(), + }, + returns: v.null(), + handler: (ctx, args) => rebuild.abortGeneration(ctx, config, args), +}); +export const purgeGeneration = internalMutation({ + args: { generationId: v.id("generations"), operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.purgeGeneration(ctx, config, args), +}); +export const fillTenants = internalMutation({ + args: { operator: v.string() }, + returns: v.null(), + handler: (ctx, args) => rebuild.fillTenants(ctx, config, args), +}); +export const backfillBatch = internalMutation({ + args: batchArgs, + returns: v.null(), + handler: (ctx, args) => rebuild.backfillBatch(ctx, config, args), +}); +export const verifyBatch = internalMutation({ + args: batchArgs, + returns: v.null(), + handler: (ctx, args) => rebuild.verifyBatch(ctx, config, args), +}); +export const purgeBatch = internalMutation({ + args: batchArgs, + returns: v.null(), + handler: (ctx, args) => rebuild.purgeBatch(ctx, config, args), +}); +export const fillTenantsBatch = internalMutation({ + args: { fence: v.number() }, + returns: v.null(), + handler: (ctx, args) => rebuild.fillTenantsBatch(ctx, config, args), +}); +export const getGenerations = internalQuery({ + args: { readModel: v.string() }, + returns: v.array( + v.object({ generation: generationDoc, progress: progressDoc }), + ), + handler: (ctx, args) => rebuild.getGenerations(ctx, config, args), +}); diff --git a/fixture/convex/summaries.ts b/fixture/convex/summaries.ts index ec37e1e..c22a51f 100644 --- a/fixture/convex/summaries.ts +++ b/fixture/convex/summaries.ts @@ -33,13 +33,15 @@ export const documentSummary: ReadModel = { name: "documentSummary", table: "documentSummaries", rowBudgetBytes: defaultRowBudgetBytes, - projection: { - version: 1, - keyOf: (_tenantId, document) => document.documentId, - project: (_tenantId, { documentId, status, title }) => ({ - documentId, - status, - title, - }), - }, + projections: [ + { + version: 1, + keyOf: (_tenantId, document) => document.documentId, + project: (_tenantId, { documentId, status, title }) => ({ + documentId, + status, + title, + }), + }, + ], }; diff --git a/fixture/convex/summarizedTwice.ts b/fixture/convex/summarizedTwice.ts index aca645d..8f7bc21 100644 --- a/fixture/convex/summarizedTwice.ts +++ b/fixture/convex/summarizedTwice.ts @@ -11,7 +11,7 @@ import { components } from "./_generated/api.js"; import { permissions } from "./depotCommands.js"; import { documentTitle } from "./documentTitles.js"; import { documentSummary } from "./summaries.js"; -const documentSource = { contextId: "depot", streamType: "document" }; +export const documentSource = { contextId: "depot", streamType: "document" }; const createInput = v.object({ documentId: v.string(), title: v.string() }); const createResult = v.object({ documentId: v.string() }); export const createTwiceSummarizedDocumentDeclaration: CommandDeclaration< diff --git a/src/read-model/generations.ts b/src/read-model/generations.ts index 9bd52e8..bcf19bf 100644 --- a/src/read-model/generations.ts +++ b/src/read-model/generations.ts @@ -1,54 +1,11 @@ -// The registry's first activation and the reads a command and a query make of it -// (spec:application.generation-registry). None is registered here. -import type { Actor } from "../command/actor-and-scope.js"; -import type { AnyReadModel, WritableGeneration } from "./projection.js"; +// The registry reads a command and a query make. None is registered here. +import type { WritableGeneration } from "./projection.js"; import { registryReader, - registryWriter, type Generation, type ReadModelDataModel, type RegistryReader, - type RegistryWriter, } from "./tables.js"; -// Makes generation 1 of a read model active. Legal only when the read model has no generation row in -// any state. It reads no source: a read model added to a deployment that already has history is a -// rebuild's work. -export async function activateFirstGeneration< - DataModel extends ReadModelDataModel, ->( - ctx: RegistryWriter, - readModel: AnyReadModel, - startedBy: Actor, -): Promise { - const db = registryWriter(ctx); - const existing = await db - .query("generations") - .withIndex("by_read_model", (q) => q.eq("readModel", readModel.name)) - .first(); - if (existing !== null) - throw new Error( - `Read model ${readModel.name} already has generation ${existing.generation}, which is ${existing.state}`, - ); - const now = Date.now(); - await db.insert("generations", { - readModel: readModel.name, - generation: 1, - projectionVersion: readModel.projection.version, - state: "active", - pauseRequired: false, - batchSize: 0, - fence: 0, - batchesDone: 0, - rowsWritten: 0, - rowsSkipped: 0, - misses: 0, - startedAt: now, - startedBy, - switchedAt: now, - updatedAt: now, - }); - return 1; -} // The generations a command writes: the active one first, then the one in building, verifying or // verified, each with its role. Two indexed reads: the state ranges active to building and verified // to verifying hold no other state, and each holds at most two rows of a sound registry. @@ -90,7 +47,9 @@ export async function generationsToWrite( }); return [ ...active.map((row) => writable("active", row)), - ...building.map((row) => writable("building", row)), + ...building.map((row) => + writable(row.state === "building" ? "building" : "verifying", row), + ), ]; } // The generation a query reads, or undefined when the read model has none active. One indexed read. diff --git a/src/read-model/index.ts b/src/read-model/index.ts index 1f088f6..cbfee0e 100644 --- a/src/read-model/index.ts +++ b/src/read-model/index.ts @@ -1,11 +1,8 @@ -// The read-model library: what the parent uses to declare, activate, write and read a read model. -export { - activateFirstGeneration, - activeGeneration, - generationsToWrite, -} from "./generations.js"; +// The read-model library: what the parent uses to declare, rebuild, write and read a read model. +export { activeGeneration, generationsToWrite } from "./generations.js"; export { applyProjection, + projectionOf, defaultRowBudgetBytes, limitRowBudgetBytes, } from "./projection.js"; @@ -22,6 +19,7 @@ export { limitReadModelList, readModelView } from "./queries.js"; export { batchCursorValidator, generationStateValidator, + progressPassValidator, readModelTables, rowConventions, } from "./tables.js"; @@ -39,3 +37,30 @@ export { writeReadModels, } from "./write.js"; export type { ReadModelBinding, SourceRef } from "./write.js"; +export { + startGeneration, + interruptGeneration, + resumeGeneration, + switchGeneration, + rollbackGeneration, + abortGeneration, + purgeGeneration, + fillTenants, + backfillBatch, + verifyBatch, + purgeBatch, + fillTenantsBatch, + getGenerations, + batchSizeFor, + rebuildBatchCeiling, + limitPurgeBatch, + resumeChain, + limitTenantFillBatch, + limitGenerationsListed, +} from "./rebuild.js"; +export type { + RebuildConfig, + RebuildTarget, + BatchRef, + FillBatchRef, +} from "./rebuild.js"; diff --git a/src/read-model/projection.ts b/src/read-model/projection.ts index 5a311e4..f57cee1 100644 --- a/src/read-model/projection.ts +++ b/src/read-model/projection.ts @@ -16,8 +16,22 @@ export type ReadModel> = { name: string; table: string; rowBudgetBytes: number; - projection: Projection; + projections: readonly [Projection, ...Projection[]]; }; +// The declared projection for one generation's version. +export function projectionOf>( + readModel: ReadModel, + version: number, +): Projection { + const projection = readModel.projections.find( + (item) => item.version === version, + ); + if (projection === undefined) + throw new Error( + `Read model ${readModel.name} declares no projection of version ${version}`, + ); + return projection; +} // A read model of any DTO and row. Both are read and written, so only any fits all. // eslint-disable-next-line @typescript-eslint/no-explicit-any export type AnyReadModel = ReadModel; @@ -26,13 +40,18 @@ export const defaultRowBudgetBytes = 16384; export const limitRowBudgetBytes = 65536; // One generation a command writes, with the role the write rule decides by. export type WritableGeneration = { - role: "active" | "building"; + role: "active" | "building" | "verifying"; generation: number; projectionVersion: number; }; -export type ApplyMode = "live-created" | "live-updated"; +export type ApplyMode = "live-created" | "live-updated" | "backfill"; export type ApplyResult = - "inserted" | "updated" | "deleted" | "skipped-missing" | "unchanged"; + | "inserted" + | "updated" + | "deleted" + | "skipped-missing" + | "skipped-newer" + | "unchanged"; export type ApplyInput = { tenantId: string; dto: D; @@ -51,7 +70,7 @@ export async function applyProjection< readModel: ReadModel, input: ApplyInput, ): Promise { - const { name, rowBudgetBytes, projection } = readModel; + const { name, rowBudgetBytes } = readModel; if (rowBudgetBytes > limitRowBudgetBytes) throw new Error( `Read model ${name} declares a row budget of ${rowBudgetBytes} bytes, above ${limitRowBudgetBytes}`, @@ -59,16 +78,36 @@ export async function applyProjection< const db = rowWriter(ctx); const table = rowsOf(readModel.table); const { tenantId, versions } = input; - const key = projection.keyOf(tenantId, input.dto); - const fields = projection.project(tenantId, input.dto, versions); const results: ApplyResult[] = []; - for (const { role, generation } of input.generations) { + for (const { role, generation, projectionVersion } of input.generations) { + const projection = projectionOf(readModel, projectionVersion); + const key = projection.keyOf(tenantId, input.dto); + const fields = projection.project(tenantId, input.dto, versions); const existing = await db .query(table) .withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("key", key), ) .unique(); + if ( + input.mode === "backfill" && + existing !== null && + !versions.some((inputVersion) => { + const recorded = existing.sourceVersions.find( + (version) => + version.tenantId === inputVersion.tenantId && + version.contextId === inputVersion.contextId && + version.streamType === inputVersion.streamType && + version.streamId === inputVersion.streamId, + ); + return ( + recorded === undefined || recorded.version < inputVersion.version + ); + }) + ) { + results.push("skipped-newer"); + continue; + } if (fields === null) { if (existing === null) results.push("unchanged"); else { diff --git a/src/read-model/rebuild.ts b/src/read-model/rebuild.ts new file mode 100644 index 0000000..2133932 --- /dev/null +++ b/src/read-model/rebuild.ts @@ -0,0 +1,747 @@ +// The operator entries and bounded batch chains of the online read-model rebuild. +import type { + FunctionReference, + GenericMutationCtx, + GenericQueryCtx, + PaginationResult, +} from "convex/server"; +import type { GenericId } from "convex/values"; +import { assertOperator } from "../command/actor-and-scope.js"; +import { nextTenant } from "../command/authority.js"; +import type { CommandDataModel } from "../command/tables.js"; +import type { ListArgs } from "../context/queries.js"; +import { utf8Length } from "../context/text.js"; +import { gateAllows, restoreDoorClosed, resumeScope } from "../gate/gate.js"; +import type { GateChangeDataModel } from "../gate/tables.js"; +import { applyProjection, projectionOf, type ReadModel } from "./projection.js"; +import { + rowWriter, + rowsOf, + type Generation, + type ReadModelDataModel, +} from "./tables.js"; +import type { SourceRef } from "./write.js"; +// A per-entity read model, its source, and the source's enumeration query. +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export type RebuildTarget = { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + readModel: ReadModel; + source: SourceRef; + list: FunctionReference<"query", "internal", ListArgs, PaginationResult>; +}; +export type BatchRef = FunctionReference< + "mutation", + "internal", + { generationId: GenericId<"generations">; fence: number }, + null +>; +export type FillBatchRef = FunctionReference< + "mutation", + "internal", + { fence: number }, + null +>; +export type RebuildConfig = { + targets: readonly RebuildTarget[]; + refs: { + backfillBatch: BatchRef; + verifyBatch: BatchRef; + purgeBatch: BatchRef; + fillTenantsBatch: FillBatchRef; + }; +}; +type RebuildDataModel = ReadModelDataModel & + CommandDataModel & + GateChangeDataModel; +type MutationCtx = GenericMutationCtx; +type GenerationArgs = { + generationId: GenericId<"generations">; + operator: string; +}; +type BatchArgs = { generationId: GenericId<"generations">; fence: number }; +// The composition holds all these tables; database helpers are not covariant in their model. +const mutationContext = ( + ctx: GenericMutationCtx, +) => ctx as unknown as MutationCtx; +export const limitTenantFillBatch = 8; +export const limitGenerationsListed = 20; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export const rebuildBatchCeiling = (readModel: ReadModel): number => + Math.min(100, Math.floor(4194304 / (2 * readModel.rowBudgetBytes))); +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export const limitPurgeBatch = (readModel: ReadModel): number => + Math.min(500, Math.floor(4194304 / readModel.rowBudgetBytes)); +export function batchSizeFor( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + readModel: ReadModel, + requested?: number, +): number { + if ( + requested !== undefined && + (!Number.isInteger(requested) || requested < 1) + ) + throw new Error( + `The batch size must be a whole number of at least 1, and ${requested} is not`, + ); + return Math.min( + requested ?? rebuildBatchCeiling(readModel), + rebuildBatchCeiling(readModel), + ); +} +function targetFor(config: RebuildConfig, name: string): RebuildTarget { + const target = config.targets.find( + ({ readModel }) => readModel.name === name, + ); + if (target === undefined) + throw new Error(`This deployment declares no read model ${name}`); + return target; +} +async function inFlight(ctx: MutationCtx, name: string) { + const building = await ctx.db + .query("generations") + .withIndex("by_read_model_state", (q) => + q.eq("readModel", name).eq("state", "building"), + ) + .first(); + if (building !== null) return building; + const verifying = await ctx.db + .query("generations") + .withIndex("by_read_model_state", (q) => + q + .eq("readModel", name) + .gte("state", "verified") + .lte("state", "verifying"), + ) + .first(); + return verifying ?? undefined; +} +function flightError(row: Generation) { + return new Error( + `Read model ${row.readModel} already has generation ${row.generation} in ${row.state}`, + ); +} +async function loadGeneration( + ctx: MutationCtx, + generationId: GenericId<"generations">, +) { + const generation = await ctx.db.get("generations", generationId); + if (generation === null) throw new Error(`No generation ${generationId}`); + return generation; +} +async function load(ctx: MutationCtx, generationId: GenericId<"generations">) { + const generation = await loadGeneration(ctx, generationId); + const progress = await ctx.db + .query("generationProgress") + .withIndex("by_generation", (q) => q.eq("generationId", generationId)) + .unique(); + if (progress === null) + throw new Error(`No progress for generation ${generationId}`); + return { generation, progress }; +} +const description = (row: Generation) => + `Generation ${row.generation} of ${row.readModel}`; +async function reopenGates( + ctx: MutationCtx, + generationId: GenericId<"generations">, + operator: string, +) { + const gate = await ctx.db + .query("maintenanceGates") + .withIndex("by_key", (q) => q.eq("key", "gates")) + .unique(); + for (const entry of gate?.closed ?? []) + if (entry.generationId === generationId) + await resumeScope(ctx, entry.scopeKey, operator); +} +export async function startGeneration( + parentCtx: GenericMutationCtx, + config: RebuildConfig, + args: { + readModel: string; + projectionVersion?: number; + batchSize?: number; + operator: string; + }, +): Promise> { + const operator = assertOperator(args.operator); + const ctx = mutationContext(parentCtx); + const target = targetFor(config, args.readModel); + const projectionVersion = + args.projectionVersion ?? target.readModel.projections[0].version; + projectionOf(target.readModel, projectionVersion); + const batchSize = batchSizeFor(target.readModel, args.batchSize); + const flight = await inFlight(ctx, args.readModel); + if (flight !== undefined) throw flightError(flight); + if ( + (await nextTenant(ctx, null)) === null && + (await ctx.db.query("grants").first()) !== null + ) + throw new Error( + "The tenant list is empty while grants exist; fillTenants fills it from the grants table", + ); + const fill = await ctx.db.query("tenantFill").first(); + if (fill?.pass === "fill") + throw new Error( + "The tenant list is being filled; startGeneration waits until fillTenants has ended", + ); + const last = await ctx.db + .query("generations") + .withIndex("by_read_model", (q) => q.eq("readModel", args.readModel)) + .order("desc") + .first(); + const now = Date.now(); + const generationId = await ctx.db.insert("generations", { + readModel: args.readModel, + generation: (last?.generation ?? 0) + 1, + projectionVersion, + state: "building", + pauseRequired: false, + fence: 1, + startedAt: now, + startedBy: operator, + changedAt: now, + changedBy: operator, + }); + await ctx.db.insert("generationProgress", { + generationId, + pass: "backfill", + batchSize, + cursor: null, + batchesDone: 0, + rowsWritten: 0, + rowsSkipped: 0, + misses: 0, + rowsPurged: 0, + updatedAt: now, + }); + await ctx.scheduler.runAfter(0, config.refs.backfillBatch, { + generationId, + fence: 1, + }); + return generationId; +} +export async function interruptGeneration( + parentCtx: GenericMutationCtx, + _config: RebuildConfig, + args: GenerationArgs, +): Promise { + const operator = assertOperator(args.operator); + const ctx = mutationContext(parentCtx); + const { generation, progress } = await load(ctx, args.generationId); + if (progress.pass === "idle") + throw new Error(`${description(generation)} has no batch to interrupt`); + const now = Date.now(); + await ctx.db.patch(generation._id, { + fence: generation.fence + 1, + changedAt: now, + changedBy: operator, + }); + await ctx.db.patch(progress._id, { + lastError: `interrupted by ${operator}`, + updatedAt: now, + }); + return null; +} +export async function resumeChain( + parentCtx: GenericMutationCtx, + config: RebuildConfig, + generationId: GenericId<"generations">, + operator: string, +): Promise { + const ctx = mutationContext(parentCtx); + const { generation, progress } = await load(ctx, generationId); + if (progress.pass === "idle") + throw new Error(`${description(generation)} has no batch to resume`); + const fence = generation.fence + 1; + const now = Date.now(); + await ctx.db.patch(generationId, { + fence, + changedAt: now, + changedBy: operator, + }); + await ctx.db.patch(progress._id, { lastError: undefined, updatedAt: now }); + const ref = + progress.pass === "backfill" + ? config.refs.backfillBatch + : progress.pass === "verify" + ? config.refs.verifyBatch + : config.refs.purgeBatch; + await ctx.scheduler.runAfter(0, ref, { generationId, fence }); +} +export async function resumeGeneration( + ctx: GenericMutationCtx, + config: RebuildConfig, + args: GenerationArgs, +): Promise { + const operator = assertOperator(args.operator); + await resumeChain(ctx, config, args.generationId, operator); + return null; +} +export async function switchGeneration( + parentCtx: GenericMutationCtx, + _config: RebuildConfig, + args: GenerationArgs & { rollbackPeriodMs?: number }, +): Promise { + const operator = assertOperator(args.operator); + const ctx = mutationContext(parentCtx); + const generation = await loadGeneration(ctx, args.generationId); + if (generation.state !== "verified") + throw new Error( + `${description(generation)} is ${generation.state}; switchGeneration needs verified`, + ); + const period = args.rollbackPeriodMs; + if ( + period !== undefined && + (!Number.isInteger(period) || period < 0 || period > 2592000000) + ) + throw new Error( + `The rollback period must be between 0 and 2592000000 ms, and ${period} is not`, + ); + const now = Date.now(); + const active = await ctx.db + .query("generations") + .withIndex("by_read_model_state", (q) => + q.eq("readModel", generation.readModel).eq("state", "active"), + ) + .unique(); + if (active !== null) + await ctx.db.patch(active._id, { + state: "retired", + retiredAt: now, + retireAfter: now + (period ?? 604800000), + changedAt: now, + changedBy: operator, + }); + await ctx.db.patch(generation._id, { + state: "active", + switchedAt: now, + retiredAt: undefined, + retireAfter: undefined, + changedAt: now, + changedBy: operator, + }); + if (generation.pauseRequired) + await reopenGates(ctx, generation._id, operator); + return null; +} +export async function rollbackGeneration( + parentCtx: GenericMutationCtx, + config: RebuildConfig, + args: GenerationArgs, +): Promise { + const operator = assertOperator(args.operator); + const ctx = mutationContext(parentCtx); + const { generation, progress } = await load(ctx, args.generationId); + if (generation.state !== "retired") + throw new Error( + `${description(generation)} is ${generation.state}; rollbackGeneration needs retired`, + ); + const now = Date.now(); + if (now >= generation.retireAfter!) + throw new Error(`${description(generation)} is past its rollback period`); + if (progress.pass === "purge") + throw new Error(`${description(generation)} is already being purged`); + if (generation.pauseRequired) + throw new Error( + `${description(generation)} needs a write pause and cannot be rolled back`, + ); + const flight = await inFlight(ctx, generation.readModel); + if (flight !== undefined) throw flightError(flight); + projectionOf( + targetFor(config, generation.readModel).readModel, + generation.projectionVersion, + ); + const fence = generation.fence + 1; + await ctx.db.patch(generation._id, { + state: "verifying", + fence, + retiredAt: undefined, + retireAfter: undefined, + changedAt: now, + changedBy: operator, + }); + await ctx.db.patch(progress._id, { + pass: "verify", + cursor: null, + misses: 0, + lastError: undefined, + updatedAt: now, + }); + await ctx.scheduler.runAfter(0, config.refs.verifyBatch, { + generationId: generation._id, + fence, + }); + return null; +} +export async function abortGeneration( + parentCtx: GenericMutationCtx, + _config: RebuildConfig, + args: GenerationArgs & { reason: string }, +): Promise { + const operator = assertOperator(args.operator); + if (utf8Length(args.reason) < 1 || utf8Length(args.reason) > 256) + throw new Error("The reason must be between 1 and 256 bytes"); + const ctx = mutationContext(parentCtx); + const { generation, progress } = await load(ctx, args.generationId); + if ( + generation.state !== "building" && + generation.state !== "verifying" && + generation.state !== "verified" + ) + throw new Error( + `${description(generation)} is ${generation.state}; abortGeneration needs building, verifying or verified`, + ); + const now = Date.now(); + await ctx.db.patch(generation._id, { + state: "aborted", + fence: generation.fence + 1, + changedAt: now, + changedBy: operator, + }); + await ctx.db.patch(progress._id, { + pass: "idle", + lastError: `aborted: ${args.reason}`, + updatedAt: now, + }); + await reopenGates(ctx, generation._id, operator); + return null; +} +export async function purgeGeneration( + parentCtx: GenericMutationCtx, + config: RebuildConfig, + args: GenerationArgs, +): Promise { + const operator = assertOperator(args.operator); + const ctx = mutationContext(parentCtx); + const { generation, progress } = await load(ctx, args.generationId); + if (generation.state !== "retired" && generation.state !== "aborted") + throw new Error( + `${description(generation)} is ${generation.state}; purgeGeneration needs retired or aborted`, + ); + const now = Date.now(); + if (generation.state === "retired" && now < generation.retireAfter!) + throw new Error(`${description(generation)} is inside its rollback period`); + if (progress.pass === "purge") + throw new Error( + `${description(generation)} is already being purged; resumeGeneration continues it`, + ); + const fence = generation.fence + 1; + await ctx.db.patch(generation._id, { + fence, + changedAt: now, + changedBy: operator, + }); + await ctx.db.patch(progress._id, { + pass: "purge", + cursor: null, + lastError: undefined, + updatedAt: now, + }); + await ctx.scheduler.runAfter(0, config.refs.purgeBatch, { + generationId: generation._id, + fence, + }); + return null; +} +// All three generation batches share the fence, tenant cursor, gate and checkpoint rules. +async function generationBatch( + ctx: MutationCtx, + config: RebuildConfig, + args: BatchArgs, + pass: "backfill" | "verify" | "purge", +): Promise { + const generation = await ctx.db.get("generations", args.generationId); + if (generation === null) return null; + const progress = await ctx.db + .query("generationProgress") + .withIndex("by_generation", (q) => q.eq("generationId", args.generationId)) + .unique(); + const stateMatches = + pass === "backfill" + ? generation.state === "building" + : pass === "verify" + ? generation.state === "verifying" + : generation.state === "retired" || generation.state === "aborted"; + if ( + !stateMatches || + progress === null || + progress.pass !== pass || + generation.fence !== args.fence + ) + return null; + const tenantId = progress.cursor?.tenantId ?? (await nextTenant(ctx, null)); + const gate = await gateAllows( + ctx, + tenantId === null ? ["all"] : ["all", `tenant:${tenantId}`], + args.generationId, + ); + if (!gate.allowed) { + await ctx.db.patch(progress._id, { + lastError: `write paused for ${gate.scopeKey}: ${gate.reason}`, + updatedAt: Date.now(), + }); + return null; + } + const target = targetFor(config, generation.readModel); + let cursor = progress.cursor; + let done = tenantId === null; + let { rowsWritten, rowsSkipped, misses, rowsPurged } = progress; + if (tenantId !== null) { + if (pass === "purge") { + const size = Math.min( + progress.batchSize, + limitPurgeBatch(target.readModel), + ); + const db = rowWriter(ctx); + const rows = await db + .query(rowsOf(target.readModel.table)) + .withIndex("by_key", (q) => + q.eq("tenantId", tenantId).eq("generation", generation.generation), + ) + .take(size); + for (const row of rows) await db.delete(row._id); + rowsPurged += rows.length; + const next = + rows.length === size ? tenantId : await nextTenant(ctx, tenantId); + done = next === null; + cursor = next === null ? null : { tenantId: next, pageCursor: null }; + } else { + const page = await ctx.runQuery(target.list, { + tenantId, + includeDeleted: true, + paginationOpts: { + numItems: progress.batchSize, + cursor: progress.cursor?.pageCursor ?? null, + }, + }); + for (const dto of page.page) { + const [result] = await applyProjection(ctx, target.readModel, { + tenantId, + dto, + versions: [dto.version], + mode: "backfill", + generations: [ + { + role: generation.state === "building" ? "building" : "verifying", + generation: generation.generation, + projectionVersion: generation.projectionVersion, + }, + ], + }); + if ( + result === "inserted" || + result === "updated" || + result === "deleted" + ) { + if (pass === "backfill") rowsWritten += 1; + else misses += 1; + } else rowsSkipped += 1; + } + if (page.isDone) { + const next = await nextTenant(ctx, tenantId); + done = next === null; + cursor = next === null ? null : { tenantId: next, pageCursor: null }; + } else cursor = { tenantId, pageCursor: page.continueCursor }; + } + } + const now = Date.now(); + const counts = { + batchesDone: progress.batchesDone + 1, + ...(!done ? { lastError: undefined } : {}), + updatedAt: now, + }; + if (pass === "purge") { + await ctx.db.patch(progress._id, { + ...counts, + cursor: done ? null : cursor, + rowsPurged, + ...(done ? { pass: "idle" as const } : {}), + }); + if (done) + await ctx.db.patch(generation._id, { + state: "purged", + retiredAt: undefined, + retireAfter: undefined, + }); + else await ctx.scheduler.runAfter(0, config.refs.purgeBatch, args); + } else { + await ctx.db.patch(progress._id, { + ...counts, + rowsSkipped, + ...(pass === "backfill" + ? { rowsWritten, ...(done ? { misses: 0 } : {}) } + : { misses }), + cursor: done ? null : cursor, + ...(done + ? { + pass: pass === "backfill" ? ("verify" as const) : ("idle" as const), + } + : {}), + }); + if (done) + await ctx.db.patch(generation._id, { + state: pass === "backfill" ? "verifying" : "verified", + }); + if (pass === "backfill") + await ctx.scheduler.runAfter( + 0, + done ? config.refs.verifyBatch : config.refs.backfillBatch, + args, + ); + else if (!done) + await ctx.scheduler.runAfter(0, config.refs.verifyBatch, args); + } + return null; +} +export async function backfillBatch( + ctx: GenericMutationCtx, + config: RebuildConfig, + args: BatchArgs, +): Promise { + return generationBatch(mutationContext(ctx), config, args, "backfill"); +} +export async function verifyBatch( + ctx: GenericMutationCtx, + config: RebuildConfig, + args: BatchArgs, +): Promise { + return generationBatch(mutationContext(ctx), config, args, "verify"); +} +export async function purgeBatch( + ctx: GenericMutationCtx, + config: RebuildConfig, + args: BatchArgs, +): Promise { + return generationBatch(mutationContext(ctx), config, args, "purge"); +} +export async function getGenerations( + parentCtx: GenericQueryCtx, + _config: RebuildConfig, + args: { readModel: string }, +) { + const ctx = parentCtx as unknown as GenericQueryCtx; + const generations = await ctx.db + .query("generations") + .withIndex("by_read_model", (q) => q.eq("readModel", args.readModel)) + .order("desc") + .take(limitGenerationsListed); + return Promise.all( + generations.map(async (generation) => { + const progress = await ctx.db + .query("generationProgress") + .withIndex("by_generation", (q) => q.eq("generationId", generation._id)) + .unique(); + if (progress === null) + throw new Error(`No progress for generation ${generation._id}`); + return { generation, progress }; + }), + ); +} +export async function fillTenants( + parentCtx: GenericMutationCtx, + config: RebuildConfig, + args: { operator: string }, +): Promise { + const operator = assertOperator(args.operator); + if (restoreDoorClosed()) + throw new Error("fillTenants is refused while the restore door is closed"); + const ctx = mutationContext(parentCtx); + for (const { readModel } of config.targets) { + const flight = await inFlight(ctx, readModel.name); + if (flight !== undefined) + throw new Error( + `Read model ${readModel.name} has generation ${flight.generation} in ${flight.state}; fillTenants needs no generation in flight`, + ); + } + const row = await ctx.db.query("tenantFill").first(); + const now = Date.now(); + const fence = (row?.fence ?? 0) + 1; + const counts = { + cursor: null, + batchesDone: 0, + tenantsRead: 0, + tenantsInserted: 0, + }; + if (row === null) + await ctx.db.insert("tenantFill", { + ...counts, + pass: "fill", + fence, + startedAt: now, + startedBy: operator, + changedAt: now, + changedBy: operator, + updatedAt: now, + }); + else + await ctx.db.patch(row._id, { + ...(row.pass === "idle" ? counts : {}), + pass: "fill", + fence, + lastError: undefined, + changedAt: now, + changedBy: operator, + updatedAt: now, + }); + await ctx.scheduler.runAfter(0, config.refs.fillTenantsBatch, { fence }); + return null; +} +export async function fillTenantsBatch( + parentCtx: GenericMutationCtx, + config: RebuildConfig, + args: { fence: number }, +): Promise { + const ctx = mutationContext(parentCtx); + const row = await ctx.db.query("tenantFill").first(); + if (row === null || row.pass !== "fill" || row.fence !== args.fence) + return null; + const gate = await gateAllows(ctx, ["all"]); + if (!gate.allowed) { + await ctx.db.patch(row._id, { + lastError: `write paused for ${gate.scopeKey}: ${gate.reason}`, + updatedAt: Date.now(), + }); + return null; + } + const now = Date.now(); + let { cursor, tenantsRead, tenantsInserted } = row; + let done = false; + for (let i = 0; i < limitTenantFillBatch; i++) { + const after = cursor; + const grant = await ctx.db + .query("grants") + .withIndex("by_principal", (q) => + after === null ? q : q.gt("tenantId", after), + ) + .first(); + if (grant === null) { + done = true; + break; + } + const tenant = await ctx.db + .query("tenants") + .withIndex("by_tenant", (q) => q.eq("tenantId", grant.tenantId)) + .first(); + if (tenant === null) { + await ctx.db.insert("tenants", { + tenantId: grant.tenantId, + createdAt: now, + }); + tenantsInserted += 1; + } + tenantsRead += 1; + cursor = grant.tenantId; + } + await ctx.db.patch(row._id, { + cursor: done ? null : cursor, + tenantsRead, + tenantsInserted, + batchesDone: row.batchesDone + 1, + ...(!done ? { lastError: undefined } : {}), + updatedAt: now, + ...(done ? { pass: "idle" as const } : {}), + }); + if (!done) + await ctx.scheduler.runAfter(0, config.refs.fillTenantsBatch, args); + return null; +} diff --git a/src/read-model/tables.ts b/src/read-model/tables.ts index 1a12eb9..8a87ebb 100644 --- a/src/read-model/tables.ts +++ b/src/read-model/tables.ts @@ -11,7 +11,6 @@ import { type SchemaDefinition, } from "convex/server"; import { v } from "convex/values"; -import { actorValidator } from "../command/actor-and-scope.js"; import { streamVersionValidator } from "../context/outcome.js"; export const generationStateValidator = v.union( v.literal("building"), @@ -28,6 +27,12 @@ export const batchCursorValidator = v.object({ streamId: v.optional(v.string()), eventCursor: v.optional(v.union(v.string(), v.null())), }); +export const progressPassValidator = v.union( + v.literal("backfill"), + v.literal("verify"), + v.literal("purge"), + v.literal("idle"), +); // One row per generation of a read model, spanning every tenant, so neither index leads with one. export const readModelTables = { generations: defineTable({ @@ -36,23 +41,53 @@ export const readModelTables = { projectionVersion: v.number(), state: generationStateValidator, pauseRequired: v.boolean(), - batchSize: v.number(), fence: v.number(), - cursor: v.optional(batchCursorValidator), - verifyCursor: v.optional(batchCursorValidator), - batchesDone: v.number(), - rowsWritten: v.number(), - rowsSkipped: v.number(), - misses: v.number(), startedAt: v.number(), - startedBy: actorValidator, + startedBy: v.string(), + changedAt: v.number(), + changedBy: v.string(), switchedAt: v.optional(v.number()), + retiredAt: v.optional(v.number()), retireAfter: v.optional(v.number()), - lastError: v.optional(v.string()), - updatedAt: v.number(), }) .index("by_read_model", ["readModel", "generation"]) .index("by_read_model_state", ["readModel", "state"]), + generationProgress: defineTable({ + generationId: v.id("generations"), + pass: progressPassValidator, + batchSize: v.number(), + cursor: v.union(batchCursorValidator, v.null()), + batchesDone: v.number(), + rowsWritten: v.number(), + rowsSkipped: v.number(), + misses: v.number(), + rowsPurged: v.number(), + lastError: v.optional(v.string()), + updatedAt: v.number(), + }).index("by_generation", ["generationId"]), + projectionMarkers: defineTable({ + tenantId: v.string(), + readModel: v.string(), + generation: v.number(), + entityKey: v.string(), + aggregateKey: v.string(), + contribution: v.number(), + sourceVersions: v.array(streamVersionValidator), + }).index("by_entity", ["tenantId", "readModel", "generation", "entityKey"]), + tenantFill: defineTable({ + pass: v.union(v.literal("fill"), v.literal("idle")), + fence: v.number(), + cursor: v.union(v.string(), v.null()), + batchesDone: v.number(), + tenantsRead: v.number(), + tenantsInserted: v.number(), + lastError: v.optional(v.string()), + startedAt: v.number(), + startedBy: v.string(), + changedAt: v.number(), + changedBy: v.string(), + updatedAt: v.number(), + }), }; // Spread into every per-entity read-model table, which also declares // .index("by_key", ["tenantId", "generation", "key"]). From ab97ed25d42cab0c6d8f8a3f85bd195d6347cdd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:02:45 +0200 Subject: [PATCH 078/155] build: 3b comments that say what each rebuild export and read-model module is Co-Authored-By: Claude Opus 5.5 --- example/convex/readModels.ts | 3 ++- fixture/convex/documentTitles.ts | 5 +++-- fixture/convex/readModels.ts | 3 ++- src/read-model/rebuild.ts | 10 ++++++++++ src/read-model/tables.ts | 3 ++- 5 files changed, 19 insertions(+), 5 deletions(-) diff --git a/example/convex/readModels.ts b/example/convex/readModels.ts index 8a23594..63c6f2c 100644 --- a/example/convex/readModels.ts +++ b/example/convex/readModels.ts @@ -1,4 +1,5 @@ -// The parent list over the read model in its active generation. +// The parent list over the order summary in its active generation. A read model is installed through its +// first rebuild, in rebuild.ts. import { paginationOptsValidator, paginationResultValidator, diff --git a/fixture/convex/documentTitles.ts b/fixture/convex/documentTitles.ts index f332d23..10ec461 100644 --- a/fixture/convex/documentTitles.ts +++ b/fixture/convex/documentTitles.ts @@ -1,7 +1,8 @@ // The fixture composition's second read model: a title row per depot document. Only -// CreateTwiceSummarizedDocument binds it, after the document summary, and its projection throws for +// CreateTwiceSummarizedDocument binds it, after the document summary, and its projections throw for // the fault title, so a failure after step 9 has written the summary row runs on the fixture -// composition and no other command meets the fault. +// composition and no other command meets the fault. Version 2 projects the deleted document to no +// row; version 1 projects every document. import { v, type Infer } from "convex/values"; import { defaultRowBudgetBytes, diff --git a/fixture/convex/readModels.ts b/fixture/convex/readModels.ts index cfee8ca..54097a5 100644 --- a/fixture/convex/readModels.ts +++ b/fixture/convex/readModels.ts @@ -1,4 +1,5 @@ -// The parent list over the read model in its active generation. +// The parent list over the document summary in its active generation. A read model is installed through its +// first rebuild, in rebuild.ts. import { paginationOptsValidator, paginationResultValidator, diff --git a/src/read-model/rebuild.ts b/src/read-model/rebuild.ts index 2133932..7a028d3 100644 --- a/src/read-model/rebuild.ts +++ b/src/read-model/rebuild.ts @@ -29,18 +29,21 @@ export type RebuildTarget = { source: SourceRef; list: FunctionReference<"query", "internal", ListArgs, PaginationResult>; }; +// A composition's own registered batch of a generation, which a batch schedules by reference. export type BatchRef = FunctionReference< "mutation", "internal", { generationId: GenericId<"generations">; fence: number }, null >; +// A composition's own registered tenant fill batch. export type FillBatchRef = FunctionReference< "mutation", "internal", { fence: number }, null >; +// The read models a composition can rebuild and its four registered batches. export type RebuildConfig = { targets: readonly RebuildTarget[]; refs: { @@ -63,14 +66,20 @@ type BatchArgs = { generationId: GenericId<"generations">; fence: number }; const mutationContext = ( ctx: GenericMutationCtx, ) => ctx as unknown as MutationCtx; +// The tenants one fill batch takes from the grants. export const limitTenantFillBatch = 8; +// The generations one getGenerations answer lists, the newest first. export const limitGenerationsListed = 20; +// The most entities one backfill or verify batch reads, from the read model's row budget. // eslint-disable-next-line @typescript-eslint/no-explicit-any export const rebuildBatchCeiling = (readModel: ReadModel): number => Math.min(100, Math.floor(4194304 / (2 * readModel.rowBudgetBytes))); +// The most rows one purge batch deletes, from the read model's row budget. // eslint-disable-next-line @typescript-eslint/no-explicit-any export const limitPurgeBatch = (readModel: ReadModel): number => Math.min(500, Math.floor(4194304 / readModel.rowBudgetBytes)); +// The batch size startGeneration stores: the request, refused below 1 or fractional, lowered to the +// ceiling, or the ceiling when none is asked. export function batchSizeFor( // eslint-disable-next-line @typescript-eslint/no-explicit-any readModel: ReadModel, @@ -242,6 +251,7 @@ export async function interruptGeneration( }); return null; } +// Bumps the fence and schedules the batch of the pass the progress row names, in the caller's mutation. export async function resumeChain( parentCtx: GenericMutationCtx, config: RebuildConfig, diff --git a/src/read-model/tables.ts b/src/read-model/tables.ts index 8a87ebb..f243660 100644 --- a/src/read-model/tables.ts +++ b/src/read-model/tables.ts @@ -1,4 +1,5 @@ -// The parent's generation registry of spec:application.generation-registry and the conventions every +// The parent's generation registry of spec:application.generation-registry, with its progress rows, +// aggregate markers and the tenant fill's row, and the conventions every // per-entity read-model row follows (spec:application.projection-contract), as fragments the parent's // schema.ts spreads into defineSchema. import { From 18c972de229e5b171a2c4f8b9d7d144273cf3954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:03:08 +0200 Subject: [PATCH 079/155] build: 3b online rebuild proof --- tests/pure/rebuild-support.ts | 170 +++ tests/pure/rebuild.test.ts | 93 ++ tests/simulator/rebuild-support.ts | 250 ++++ tests/simulator/rebuild.test.ts | 2225 ++++++++++++++++++++++++++++ tests/types/rebuild.test-d.ts | 993 +++++++++++++ 5 files changed, 3731 insertions(+) create mode 100644 tests/pure/rebuild-support.ts create mode 100644 tests/pure/rebuild.test.ts create mode 100644 tests/simulator/rebuild-support.ts create mode 100644 tests/simulator/rebuild.test.ts create mode 100644 tests/types/rebuild.test-d.ts diff --git a/tests/pure/rebuild-support.ts b/tests/pure/rebuild-support.ts new file mode 100644 index 0000000..ae8be53 --- /dev/null +++ b/tests/pure/rebuild-support.ts @@ -0,0 +1,170 @@ +import type { StreamVersion } from "../../src/kernel/index.js"; +import type { + ApplyResult, + ReadModel, +} from "../../src/read-model/projection.js"; +import type { RowWriter } from "../../src/read-model/tables.js"; + +export const modes = ["live-created", "live-updated", "backfill"] as const; +export const roles = ["active", "building", "verifying"] as const; +export const rowStates = [ + "absent", + "older", + "equal", + "newer", + "other-stream", +] as const; +export type RowState = (typeof rowStates)[number]; +export const version = (n: number): StreamVersion => ({ + tenantId: "tenant-a", + contextId: "depot", + streamType: "document", + streamId: "document-a", + version: n, +}); +export const dto = { documentId: "document-a", title: "candidate" }; +export const readModel: ReadModel = { + name: "documentTitle", + table: "documentTitles", + rowBudgetBytes: 16384, + projections: [ + { + version: 2, + keyOf: (_tenant, d) => d.documentId, + project: (_tenant, d) => (d.title === "(deleted)" ? null : d), + }, + { + version: 1, + keyOf: (_tenant, d) => d.documentId, + project: (_tenant, d) => d, + }, + ], +}; +export const candidateRow = { + ...dto, + tenantId: "tenant-a", + generation: 1, + key: "document-a", + projectionVersion: 2, + sourceVersions: [version(3)], +}; +export function priorRow(state: RowState) { + if (state === "absent") return null; + return { + ...candidateRow, + title: "prior", + sourceVersions: [ + state === "other-stream" + ? { ...version(99), streamId: "another-document" } + : version({ older: 2, equal: 3, newer: 4 }[state]), + ], + }; +} +// projection-contract.sdp.md:64, step1:90, step3:92. These are the expected result tables, +// including a row whose versions contain no entry for the input stream. +const live: Record = { + absent: "inserted", + older: "updated", + equal: "updated", + newer: "updated", + "other-stream": "updated", +}; +const backfill: Record = { + absent: "inserted", + older: "updated", + equal: "skipped-newer", + newer: "skipped-newer", + "other-stream": "updated", +}; +const liveNull: Record = { + absent: "unchanged", + older: "deleted", + equal: "deleted", + newer: "deleted", + "other-stream": "deleted", +}; +const backfillNull: Record = { + absent: "unchanged", + older: "deleted", + equal: "skipped-newer", + newer: "skipped-newer", + "other-stream": "deleted", +}; +export const projectionCases = modes.flatMap((mode) => + roles.flatMap((role) => + rowStates.flatMap((state) => + [false, true].map((isNull) => ({ + mode, + role, + state, + isNull, + expected: isNull + ? (mode === "backfill" ? backfillNull : liveNull)[state] + : mode === "backfill" + ? backfill[state] + : mode === "live-updated" && + role === "building" && + state === "absent" + ? ("skipped-missing" as const) + : live[state], + })), + ), + ), +); + +// A single-table writer. It enforces the index predicates and preserves system fields on replace. +// The cast supplies Convex's database type, not a replacement for applyProjection. +export function memoryWriter(initial: ReturnType) { + type Stored = NonNullable & { + _id: string; + _creationTime: number; + }; + let row: Stored | null = + initial === null ? null : { ...initial, _id: "row", _creationTime: 1 }; + let writes = 0; + const db = { + query: (table: string) => { + if (table !== "documentTitles") throw new Error(table); + return { + withIndex: (index: string, select: (q: unknown) => unknown) => { + if (index !== "by_key") throw new Error(index); + const equalities: [string, unknown][] = []; + const q = { + eq: (key: string, value: unknown) => { + equalities.push([key, value]); + return q; + }, + }; + select(q); + return { + unique: async () => + row && + equalities.every( + ([key, value]) => row![key as keyof Stored] === value, + ) + ? structuredClone(row) + : null, + }; + }, + }; + }, + insert: async (_table: string, value: NonNullable) => { + writes++; + row = { ...value, _id: "row", _creationTime: 1 }; + return "row"; + }, + replace: async (_id: string, value: NonNullable) => { + writes++; + row = { ...value, _id: "row", _creationTime: 1 }; + }, + delete: async () => { + writes++; + row = null; + }, + }; + return { + ctx: { db } as unknown as RowWriter, + row: () => structuredClone(row), + writes: () => writes, + }; +} diff --git a/tests/pure/rebuild.test.ts b/tests/pure/rebuild.test.ts new file mode 100644 index 0000000..a2c9b5b --- /dev/null +++ b/tests/pure/rebuild.test.ts @@ -0,0 +1,93 @@ +import { expect, test } from "vitest"; +import { documentTitle } from "../../fixture/convex/documentTitles.js"; +import { + applyProjection, + projectionOf, +} from "../../src/read-model/projection.js"; +import { + candidateRow, + dto, + memoryWriter, + priorRow, + projectionCases, + readModel, + version, +} from "./rebuild-support.js"; + +// projection-contract.sdp.md fnProjectionOf:68, typeReadModel:67; native-harness.sdp.md fixtureReadModel:90. +test("pure: projectionOf selects the declared version and the latest projection is first", () => { + expect(readModel.projections.map((p) => p.version)).toEqual([2, 1]); + expect(documentTitle.projections.map((p) => p.version)).toEqual([2, 1]); + expect(projectionOf(readModel, 1)).toBe(readModel.projections[1]); + expect(projectionOf(readModel, 2)).toBe(readModel.projections[0]); + expect(() => projectionOf(readModel, 7)).toThrowError( + new Error("Read model documentTitle declares no projection of version 7"), + ); +}); + +// rebuild.sdp.md fnBatchSizeFor:116, limitBatchSize:146, limitPurgeBatch:147. +test.each([ + [16384, 100, 256], + [65536, 32, 64], +])( + "pure: batch and purge bounds at row budget %i", + async (rowBudgetBytes, ceiling, purge) => { + const { batchSizeFor, rebuildBatchCeiling, limitPurgeBatch } = + await import("../../src/read-model/rebuild.js"); + const model = { ...readModel, rowBudgetBytes }; + expect(rebuildBatchCeiling(model)).toBe(ceiling); + expect(limitPurgeBatch(model)).toBe(purge); + expect(batchSizeFor(model)).toBe(ceiling); + for (const n of [1, ceiling, ceiling + 1, 10000]) + expect(batchSizeFor(model, n)).toBe(Math.min(n, ceiling)); + for (const n of [0, -1, 1.5, NaN, Infinity, -Infinity]) { + expect(() => batchSizeFor(model, n)).toThrowError( + new Error( + `The batch size must be a whole number of at least 1, and ${n} is not`, + ), + ); + } + }, +); + +// projection-contract.sdp.md:64, step1:90, step2:91, step3:92, step4:93. +test.each(projectionCases)( + "pure: $mode / $role / $state / null=$isNull", + async ({ mode, role, state, isNull, expected }) => { + const memory = memoryWriter(priorRow(state)); + const before = memory.row(); + expect( + await applyProjection(memory.ctx, readModel, { + tenantId: "tenant-a", + dto: { ...dto, title: isNull ? "(deleted)" : "candidate" }, + versions: [version(3)], + mode, + generations: [{ role, generation: 1, projectionVersion: 2 }], + }), + ).toEqual([expected]); + if (expected === "inserted" || expected === "updated") { + expect(memory.row()).toStrictEqual({ + ...candidateRow, + _id: "row", + _creationTime: 1, + }); + expect(memory.writes()).toBe(1); + } else if (expected === "deleted") { + expect(memory.row()).toBeNull(); + expect(memory.writes()).toBe(1); + } else { + expect(memory.row()).toStrictEqual(before); + expect(memory.writes()).toBe(0); + } + }, +); +// rebuild.sdp.md step6:126 and verifySteps:127 expose no pure cursor function. +// The direct batch tests in tests/simulator/rebuild.test.ts prove those cursor steps. + +// rebuild.sdp.md limitTenantFillBatch:142, limitGenerationsListed:151. +test("pure: tenant fill and generation listing have their pinned bounds", async () => { + const { limitTenantFillBatch, limitGenerationsListed } = + await import("../../src/read-model/rebuild.js"); + expect(limitTenantFillBatch).toBe(8); + expect(limitGenerationsListed).toBe(20); +}); diff --git a/tests/simulator/rebuild-support.ts b/tests/simulator/rebuild-support.ts new file mode 100644 index 0000000..5a164bd --- /dev/null +++ b/tests/simulator/rebuild-support.ts @@ -0,0 +1,250 @@ +import { convexTest } from "convex-test"; +import { ConvexError, v } from "convex/values"; +import { defineSchema, defineTable } from "convex/server"; +import { expect } from "vitest"; +import { api, internal } from "../../fixture/convex/_generated/api.js"; +import type { Doc, Id } from "../../fixture/convex/_generated/dataModel.js"; +import annexSchema from "../../fixture/convex/annex/schema.js"; +import depotSchema from "../../fixture/convex/depot/schema.js"; +import schema from "../../fixture/convex/schema.js"; + +export function rebuildApp(onlyFirstActive = false) { + // The fault schema refuses activation of generation two. All registered functions and + // component schemas remain the composition's own. + const fields = schema.tables.generations.validator.fields; + const selected = onlyFirstActive + ? (defineSchema({ + ...schema.tables, + generations: defineTable( + v.union( + v.object({ + ...fields, + state: v.union( + v.literal("building"), + v.literal("verifying"), + v.literal("verified"), + v.literal("retired"), + v.literal("aborted"), + v.literal("purged"), + ), + }), + v.object({ + ...fields, + state: v.literal("active"), + generation: v.literal(1), + }), + ), + ) + .index("by_read_model", ["readModel", "generation"]) + .index("by_read_model_state", ["readModel", "state"]), + }) as unknown as typeof schema) + : schema; + const t = convexTest( + selected, + import.meta.glob("../../fixture/convex/**/*.ts"), + ); + t.registerComponent( + "annex", + annexSchema, + import.meta.glob("../../fixture/convex/annex/**/*.ts"), + ); + t.registerComponent( + "depot", + depotSchema, + import.meta.glob("../../fixture/convex/depot/**/*.ts"), + ); + return t; +} +export type App = ReturnType; +export type GenerationId = Id<"generations">; +export const operator = "rebuild operator"; +export const start = ( + t: App, + args: { + readModel?: string; + projectionVersion?: number; + batchSize?: number; + operator?: string; + } = {}, +) => + t.mutation(internal.rebuild.startGeneration, { + readModel: "documentTitle", + projectionVersion: 1, + operator, + ...args, + }); +export async function read(t: App, generationId: GenerationId) { + return t.run(async (ctx) => { + const generation = await ctx.db.get(generationId); + const progress = await ctx.db + .query("generationProgress") + .withIndex("by_generation", (q) => q.eq("generationId", generationId)) + .unique(); + if (!generation || !progress) + throw new Error("Generation and progress are required"); + return { generation, progress }; + }); +} +export const rows = (t: App) => + t.run((ctx) => ctx.db.query("documentTitles").collect()); +export const summaries = (t: App) => + t.run((ctx) => ctx.db.query("documentSummaries").collect()); +export const scheduled = (t: App) => + t.run((ctx) => ctx.db.system.query("_scheduled_functions").collect()); +export const fill = (t: App) => + t.run((ctx) => ctx.db.query("tenantFill").unique()); +export const tenants = (t: App) => + t.run((ctx) => ctx.db.query("tenants").withIndex("by_tenant").collect()); +export async function snapshot(t: App) { + return t.run(async (ctx) => ({ + generations: await ctx.db.query("generations").collect(), + progress: await ctx.db.query("generationProgress").collect(), + titles: await ctx.db.query("documentTitles").collect(), + summaries: await ctx.db.query("documentSummaries").collect(), + tenants: await ctx.db.query("tenants").collect(), + fills: await ctx.db.query("tenantFill").collect(), + scheduled: await ctx.db.system.query("_scheduled_functions").collect(), + })); +} +export async function failure(call: Promise, message: string) { + const error: unknown = await call.then( + () => { + throw new Error("Expected refusal"); + }, + (e) => e, + ); + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(ConvexError); + expect((error as Error).message).toBe(message); +} +export async function refused( + t: App, + call: () => Promise, + message: string, +) { + const before = await snapshot(t); + await failure(call(), message); + expect(await snapshot(t)).toStrictEqual(before); +} +export async function batch(t: App, generationId: GenerationId) { + const { generation, progress } = await read(t, generationId); + const args = { generationId, fence: generation.fence }; + switch (progress.pass) { + case "backfill": + return t.mutation(internal.rebuild.backfillBatch, args); + case "verify": + return t.mutation(internal.rebuild.verifyBatch, args); + case "purge": + return t.mutation(internal.rebuild.purgeBatch, args); + default: + throw new Error("No pass to run"); + } +} +export async function finish(t: App, id: GenerationId) { + for (let i = 0; i < 100; i++) { + if ((await read(t, id)).progress.pass === "idle") return; + await batch(t, id); + } + throw new Error("The pass did not finish within 100 direct batches"); +} +export async function install( + t: App, + readModel = "documentTitle", + projectionVersion = 1, +) { + const id = await start(t, { readModel, projectionVersion }); + await finish(t, id); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator, + }); + return id; +} +export const tenant = (t: App, tenantId: string) => + t.run((ctx) => ctx.db.insert("tenants", { tenantId, createdAt: Date.now() })); +export async function caller(t: App, tenantId = "tenant-a") { + await t.mutation(internal.grants.grant, { + tenantId, + principalKind: "human", + principalId: "https://rebuild.test|reader", + permission: "depot.documents", + grantedBy: operator, + }); + return t.withIdentity({ issuer: "https://rebuild.test", subject: "reader" }); +} +export async function create( + t: App, + documentId: string, + title = documentId, + tenantId = "tenant-a", + kind: "plain" | "summary" | "twice" = "plain", +) { + const user = await caller(t, tenantId); + const ref = + kind === "plain" + ? api.depotCommands.createDocument + : kind === "summary" + ? api.depotCommands.createSummarizedDocument + : api.summarizedTwice.createTwiceSummarizedDocument; + const result = await user.mutation(ref, { + tenantId, + input: { documentId, title }, + }); + expect(result.kind).toBe("applied"); + return result; +} +export async function amend( + t: App, + documentId: string, + title: string, + summary = false, +) { + const user = await caller(t); + const result = await user.mutation( + summary + ? api.depotCommands.amendSummarizedDocument + : api.depotCommands.amendDocument, + { tenantId: "tenant-a", input: { documentId, title } }, + ); + expect(result.kind).toBe("applied"); + return result; +} +export async function patchGeneration( + t: App, + id: GenerationId, + fields: Partial>, + progress: Partial> = {}, +) { + const { progress: p } = await read(t, id); + await t.run(async (ctx) => { + await ctx.db.patch(id, fields); + await ctx.db.patch(p._id, progress); + }); +} +// Refusal tests arrange otherwise unreachable or invalid states directly. Successful installs use +// startGeneration, direct batches and switchGeneration. +export async function inState( + t: App, + state: Doc<"generations">["state"], + pass: Doc<"generationProgress">["pass"] = "idle", +) { + const id = await start(t); + await patchGeneration(t, id, { state }, { pass }); + return id; +} +export async function grantsWithoutTenants(t: App, count: number) { + await t.run(async (ctx) => { + for (let i = count - 1; i >= 0; i--) { + const tenantId = `tenant-${String(i).padStart(2, "0")}`; + for (const principalId of ["one", "two"]) + await ctx.db.insert("grants", { + tenantId, + principalKind: "human", + principalId, + permission: "depot.documents", + grantedBy: operator, + grantedAt: Date.now(), + }); + } + }); +} diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts new file mode 100644 index 0000000..7a25a4e --- /dev/null +++ b/tests/simulator/rebuild.test.ts @@ -0,0 +1,2225 @@ +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { components, internal } from "../../fixture/convex/_generated/api.js"; +import { internal as exampleInternal } from "../../example/convex/_generated/api.js"; +import { deletedTitle } from "../../fixture/convex/depot/streams.js"; +import { documentTitle } from "../../fixture/convex/documentTitles.js"; +import { applyProjection } from "../../src/read-model/projection.js"; +import { + candidateRow, + dto, + priorRow, + projectionCases, + readModel, + version, +} from "../pure/rebuild-support.js"; +import { productionTest } from "./production.js"; +import { + amend, + batch, + create, + failure, + fill, + finish, + grantsWithoutTenants, + inState, + install, + operator, + patchGeneration, + read, + rebuildApp, + refused, + rows, + scheduled, + snapshot, + start, + summaries, + tenant, + tenants, + type App, + type GenerationId, +} from "./rebuild-support.js"; + +// Scheduler timers never run. Every batch below is invoked as a mutation by the test. +beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(1000); + vi.stubEnv("MAINTENANCE_MODE", ""); +}); +afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + vi.unstubAllEnvs(); +}); +const later = () => vi.setSystemTime(Date.now() + 10); +const states = [ + "building", + "verifying", + "verified", + "active", + "retired", + "aborted", + "purged", +] as const; +const without = (value: T, ...keys: (keyof T)[]) => + Object.fromEntries( + Object.entries(value).filter(([key]) => !keys.includes(key as keyof T)), + ); +const generationMessage = (state: string, entry: string, needs: string) => + `Generation 1 of documentTitle is ${state}; ${entry} needs ${needs}`; +const zeros = { + batchesDone: 0, + rowsWritten: 0, + rowsSkipped: 0, + misses: 0, + rowsPurged: 0, +}; +async function lastSchedule(t: App, name: string, args: object) { + const all = await scheduled(t); + expect(all.at(-1)).toStrictEqual({ + _id: expect.any(String), + _creationTime: expect.any(Number), + name: `rebuild:${name}`, + args: [args], + scheduledTime: Date.now(), + state: { kind: "pending" }, + }); +} + +// rebuild.sdp.md fnStartGeneration:115; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test.each([false, true])( + "convex-test: install on an empty deployment, tenant present=%s", + async (present) => { + const t = rebuildApp(); + if (present) await tenant(t, "tenant-a"); + const id = await start(t, { operator: " installer " }); + const initial = await read(t, id); + expect(initial.generation).toStrictEqual({ + _id: id, + _creationTime: expect.any(Number), + readModel: "documentTitle", + generation: 1, + projectionVersion: 1, + state: "building", + pauseRequired: false, + fence: 1, + startedAt: 1000, + changedAt: 1000, + startedBy: "installer", + changedBy: "installer", + }); + expect(initial.progress).toStrictEqual({ + _id: expect.any(String), + _creationTime: expect.any(Number), + generationId: id, + pass: "backfill", + batchSize: 100, + cursor: null, + ...zeros, + updatedAt: 1000, + }); + await lastSchedule(t, "backfillBatch", { generationId: id, fence: 1 }); + later(); + await t.mutation(internal.rebuild.backfillBatch, { + generationId: id, + fence: 1, + }); + const backfilled = await read(t, id); + expect(backfilled.generation).toStrictEqual({ + ...initial.generation, + state: "verifying", + }); + expect(backfilled.progress).toStrictEqual({ + ...initial.progress, + pass: "verify", + batchesDone: 1, + updatedAt: 1010, + }); + await lastSchedule(t, "verifyBatch", { generationId: id, fence: 1 }); + const beforeVerify = await scheduled(t); + later(); + await t.mutation(internal.rebuild.verifyBatch, { + generationId: id, + fence: 1, + }); + const verified = await read(t, id); + expect(verified.generation).toStrictEqual({ + ...initial.generation, + state: "verified", + }); + expect(verified.progress).toStrictEqual({ + ...initial.progress, + pass: "idle", + batchesDone: 2, + updatedAt: 1020, + }); + expect(await scheduled(t)).toStrictEqual(beforeVerify); + later(); + expect( + await t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator: " switch operator ", + }), + ).toBeNull(); + expect(await read(t, id)).toStrictEqual({ + generation: { + ...initial.generation, + state: "active", + switchedAt: 1030, + changedAt: 1030, + changedBy: "switch operator", + }, + progress: verified.progress, + }); + expect((await snapshot(t)).generations).toHaveLength(1); + }, +); + +// rebuild.sdp.md fnStartGeneration:115; projection-contract.sdp.md typeReadModel:67. +test("convex-test: the default projection is the first and generation numbers increase past aborted rows", async () => { + const t = rebuildApp(); + const first = await t.mutation(internal.rebuild.startGeneration, { + readModel: "documentTitle", + operator, + }); + expect((await read(t, first)).generation.projectionVersion).toBe(2); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: first, + operator, + reason: "replace", + }); + const second = await start(t); + expect((await read(t, second)).generation.generation).toBe(2); +}); + +// rebuild.sdp.md fnStartGeneration:115, fnBatchSizeFor:116, errorUnknownGeneration:143. +test("convex-test: start refusals follow target, projection, batch size, in-flight, empty list and fill order", async () => { + const t = rebuildApp(); + await grantsWithoutTenants(t, 1); + await t.mutation(internal.rebuild.fillTenants, { operator }); + await refused( + t, + () => + start(t, { readModel: "missing", projectionVersion: 77, batchSize: 0 }), + "This deployment declares no read model missing", + ); + await refused( + t, + () => start(t, { projectionVersion: 77, batchSize: 0 }), + "Read model documentTitle declares no projection of version 77", + ); + await refused( + t, + () => start(t, { batchSize: 0 }), + "The batch size must be a whole number of at least 1, and 0 is not", + ); + await refused( + t, + () => start(t), + "The tenant list is empty while grants exist; fillTenants fills it from the grants table", + ); + await tenant(t, "tenant-00"); + await refused( + t, + () => start(t), + "The tenant list is being filled; startGeneration waits until fillTenants has ended", + ); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + const id = await start(t); + // The in-flight refusal precedes both tenant-list refusals. + await t.run(async (ctx) => { + for (const row of await ctx.db.query("tenants").collect()) + await ctx.db.delete(row._id); + const f = await ctx.db.query("tenantFill").unique(); + await ctx.db.patch(f!._id, { pass: "fill" }); + }); + for (const state of ["building", "verifying", "verified"] as const) { + await patchGeneration(t, id, { state }); + await refused( + t, + () => start(t), + `Read model documentTitle already has generation 1 in ${state}`, + ); + } +}); + +// rebuild.sdp.md fnStartGeneration:115, fnBatchSizeFor:116. +test.each([0, -1, 1.5, NaN, Infinity])( + "convex-test: start refuses batch size %s with its full message", + async (batchSize) => { + const t = rebuildApp(); + await refused( + t, + () => start(t, { batchSize }), + `The batch size must be a whole number of at least 1, and ${batchSize} is not`, + ); + }, +); + +// rebuild.sdp.md fnStartGeneration:115, limitBatchSize:146. +test.each([100, 101])( + "convex-test: start stores the capped batch size for requested %i", + async (batchSize) => { + const t = rebuildApp(); + const id = await start(t, { batchSize }); + expect((await read(t, id)).progress.batchSize).toBe(100); + }, +); + +// rebuild.sdp.md errorUnknownGeneration:143, fnInterruptGeneration:130, fnResumeGeneration:132, +// fnSwitchGeneration:133, fnRollbackGeneration:134, fnAbortGeneration:135, fnPurgeGeneration:136. +test("convex-test: every generation entry refuses an unknown ID and every batch leaves it alone", async () => { + const t = rebuildApp(); + const id = await start(t); + const { progress } = await read(t, id); + await t.run(async (ctx) => { + await ctx.db.delete(id); + await ctx.db.delete(progress._id); + }); + for (const ref of [ + internal.rebuild.interruptGeneration, + internal.rebuild.resumeGeneration, + internal.rebuild.switchGeneration, + internal.rebuild.rollbackGeneration, + internal.rebuild.purgeGeneration, + ]) { + await refused( + t, + () => t.mutation(ref, { generationId: id, operator }), + `No generation ${id}`, + ); + } + await refused( + t, + () => + t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }), + `No generation ${id}`, + ); + const before = await snapshot(t); + for (const ref of [ + internal.rebuild.backfillBatch, + internal.rebuild.verifyBatch, + internal.rebuild.purgeBatch, + ]) + expect(await t.mutation(ref, { generationId: id, fence: 1 })).toBeNull(); + expect(await snapshot(t)).toStrictEqual(before); +}); + +// actor-and-scope.sdp.md fnAssertOperator:91; rebuild.sdp.md fnStartGeneration:115 through fnFillTenants:139. +test.each(["", " \n\t ", " ".repeat(513), "x".repeat(513), "é".repeat(257)])( + "convex-test: every operator entry refuses invalid operator %j before a lookup", + async (stated) => { + const t = rebuildApp(); + const id = await start(t); + const p = (await read(t, id)).progress; + await t.run(async (ctx) => { + await ctx.db.delete(id); + await ctx.db.delete(p._id); + }); + const message = + stated.trim() === "" + ? "An operator entry needs a stated operator" + : `The stated operator is ${new TextEncoder().encode(stated.trim()).length} bytes, above the limit of 512`; + const args = { generationId: id, operator: stated }; + for (const ref of [ + internal.rebuild.interruptGeneration, + internal.rebuild.resumeGeneration, + internal.rebuild.switchGeneration, + internal.rebuild.rollbackGeneration, + internal.rebuild.purgeGeneration, + ]) + await refused(t, () => t.mutation(ref, args), message); + await refused( + t, + () => + t.mutation(internal.rebuild.abortGeneration, { ...args, reason: "" }), + message, + ); + await refused( + t, + () => start(t, { readModel: "missing", operator: stated }), + message, + ); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + await refused( + t, + () => t.mutation(internal.rebuild.fillTenants, { operator: stated }), + message, + ); + }, +); + +// actor-and-scope.sdp.md fnAssertOperator:91; rebuild.sdp.md convexSurface:112. +test("convex-test: every plain entry checks the operator before touching ctx or config", async () => { + const t = rebuildApp(); + const id = await start(t); + const helpers = await import("../../src/read-model/rebuild.js"); + await t.run(async (ctx) => { + const touched = vi.fn(() => { + throw new Error("Read before assertOperator"); + }); + const guarded = new Proxy(ctx, { get: touched }); + const config = new Proxy( + {} as import("../../src/read-model/rebuild.js").RebuildConfig, + { get: touched }, + ); + const args = { generationId: id, operator: " " }; + const calls = [ + () => + helpers.startGeneration(guarded, config, { + readModel: "documentTitle", + operator: " ", + }), + () => helpers.interruptGeneration(guarded, config, args), + () => helpers.resumeGeneration(guarded, config, args), + () => helpers.switchGeneration(guarded, config, args), + () => helpers.rollbackGeneration(guarded, config, args), + () => helpers.abortGeneration(guarded, config, { ...args, reason: "" }), + () => helpers.purgeGeneration(guarded, config, args), + () => helpers.fillTenants(guarded, config, { operator: " " }), + ]; + for (const call of calls) + await failure( + Promise.resolve().then(call), + "An operator entry needs a stated operator", + ); + expect(touched).not.toHaveBeenCalled(); + }); +}); + +// rebuild.sdp.md step6:126, verifySteps:127, fnEnumerateCall:120; actor-and-scope.sdp.md fnNextTenant:89. +test("convex-test: page cursors continue, tenants cross in tenant-ID order and verify restarts at the first tenant", async () => { + const t = rebuildApp(); + await create(t, "z", "z", "tenant-z"); + await create(t, "a1"); + await create(t, "a2"); + await create(t, "a3"); + const id = await start(t, { batchSize: 2 }); + const first = await t.run((ctx) => + ctx.runQuery(components.depot.queries.document.list, { + tenantId: "tenant-a", + includeDeleted: true, + paginationOpts: { numItems: 2, cursor: null }, + }), + ); + expect(first.isDone).toBe(false); + const initial = await read(t, id); + later(); + await batch(t, id); + expect(await read(t, id)).toStrictEqual({ + generation: initial.generation, + progress: { + ...initial.progress, + cursor: { tenantId: "tenant-a", pageCursor: first.continueCursor }, + batchesDone: 1, + rowsWritten: 2, + updatedAt: 1010, + }, + }); + later(); + await batch(t, id); + expect((await read(t, id)).progress.cursor).toStrictEqual({ + tenantId: "tenant-z", + pageCursor: null, + }); + expect((await rows(t)).map((r) => r.key).sort()).toEqual(["a1", "a2", "a3"]); + later(); + await batch(t, id); + const backfilled = await read(t, id); + expect(backfilled.generation).toStrictEqual({ + ...initial.generation, + state: "verifying", + }); + expect(backfilled.progress).toStrictEqual({ + ...initial.progress, + pass: "verify", + cursor: null, + batchesDone: 3, + rowsWritten: 4, + updatedAt: 1030, + }); + later(); + await batch(t, id); + expect((await read(t, id)).progress.cursor).toStrictEqual({ + tenantId: "tenant-a", + pageCursor: first.continueCursor, + }); + expect((await read(t, id)).progress.rowsSkipped).toBe(2); +}); + +// rebuild.sdp.md step2:122, step6:126. The empty tenant is an independent tenant-list obligation. +test("convex-test: an empty tenant consumes a batch and advances to the next tenant", async () => { + const t = rebuildApp(); + await tenant(t, "tenant-a"); + await create(t, "z", "z", "tenant-z"); + const id = await start(t); + const before = await read(t, id); + later(); + await batch(t, id); + expect(await read(t, id)).toStrictEqual({ + generation: before.generation, + progress: { + ...before.progress, + cursor: { tenantId: "tenant-z", pageCursor: null }, + batchesDone: 1, + updatedAt: 1010, + }, + }); + expect(await rows(t)).toEqual([]); + await finish(t, id); + expect((await rows(t)).map((r) => [r.tenantId, r.key])).toEqual([ + ["tenant-z", "z"], + ]); +}); + +// rebuild.sdp.md rule:81, step6:126, verifySteps:127. The tenant behind the cursor is an independent obligation. +test("convex-test: a tenant inserted between crossed tenants is skipped by backfill and visited by verify", async () => { + const t = rebuildApp(); + await install(t); + await install(t, "documentSummary"); + await create(t, "a", "a", "tenant-a", "twice"); + await create(t, "m", "m", "tenant-m", "twice"); + await create(t, "z", "z", "tenant-z", "twice"); + const id = await start(t); + await batch(t, id); + await batch(t, id); + expect((await read(t, id)).progress.cursor).toStrictEqual({ + tenantId: "tenant-z", + pageCursor: null, + }); + await create(t, "h", "live", "tenant-h", "twice"); + const liveRow = (await rows(t)).find( + (r) => r.generation === 2 && r.tenantId === "tenant-h", + ); + expect(liveRow?.sourceVersions[0]?.version).toBe(1); + await batch(t, id); + expect((await read(t, id)).progress.rowsWritten).toBe(3); + expect((await read(t, id)).progress.batchesDone).toBe(3); + await batch(t, id); + expect((await read(t, id)).progress.cursor).toStrictEqual({ + tenantId: "tenant-h", + pageCursor: null, + }); + await batch(t, id); + expect((await read(t, id)).progress.rowsSkipped).toBe(2); + expect((await rows(t)).find((r) => r._id === liveRow?._id)).toStrictEqual( + liveRow, + ); +}); + +// rebuild.sdp.md step1:121, verifySteps:127, purgeSteps:128, limitOutstandingBatches:148. +test.each(["backfill", "verify", "purge"] as const)( + "convex-test: a stale fence leaves every row and schedule unchanged in %s", + async (pass) => { + const t = rebuildApp(); + await create(t, "a"); + const id = await start(t); + if (pass === "verify") await batch(t, id); + if (pass === "purge") { + await finish(t, id); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }); + await t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator, + }); + } + const fence = (await read(t, id)).generation.fence; + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator, + }); + const before = await snapshot(t); + const ref = + pass === "backfill" + ? internal.rebuild.backfillBatch + : pass === "verify" + ? internal.rebuild.verifyBatch + : internal.rebuild.purgeBatch; + await t.mutation(ref, { generationId: id, fence }); + expect(await snapshot(t)).toStrictEqual(before); + }, +); + +// rebuild.sdp.md fnInterruptGeneration:130, fnResumeChain:131, fnResumeGeneration:132; rebuild.online-rebuild-interrupt-resume.sdp.md:38. +test("convex-test: interrupt after exactly three batches and resume at the stored size with a new fence", async () => { + const t = rebuildApp(); + for (let i = 0; i < 6; i++) await create(t, `d${i}`); + const id = await start(t, { batchSize: 1 }); + for (let i = 0; i < 3; i++) { + later(); + await batch(t, id); + } + const before = await read(t, id); + const queued = await scheduled(t); + expect(before.progress.batchesDone).toBe(3); + expect(before.progress.rowsWritten).toBe(3); + later(); + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator: " interrupted ", + }); + const interrupted = await read(t, id); + expect(interrupted).toStrictEqual({ + generation: { + ...before.generation, + fence: 2, + changedAt: 1040, + changedBy: "interrupted", + }, + progress: { + ...before.progress, + lastError: "interrupted by interrupted", + updatedAt: 1040, + }, + }); + expect(await scheduled(t)).toStrictEqual(queued); + later(); + await t.mutation(internal.rebuild.resumeGeneration, { + generationId: id, + operator: " resumed ", + }); + const resumed = await read(t, id); + expect(resumed).toStrictEqual({ + generation: { + ...before.generation, + fence: 3, + changedAt: 1050, + changedBy: "resumed", + }, + progress: { ...before.progress, updatedAt: 1050 }, + }); + await lastSchedule(t, "backfillBatch", { generationId: id, fence: 3 }); + await t.mutation(internal.rebuild.backfillBatch, { + generationId: id, + fence: 3, + }); + const next = await read(t, id); + expect(next.generation).toStrictEqual(resumed.generation); + expect(next.progress.batchSize).toBe(1); + expect(next.progress.batchesDone).toBe(4); + expect(next.progress.rowsWritten).toBe(4); + expect(next.progress.cursor).not.toEqual(resumed.progress.cursor); +}); + +// rebuild.sdp.md fnInterruptGeneration:130, fnResumeChain:131. +test.each(states)( + "convex-test: an idle pass in %s cannot be interrupted or resumed", + async (state) => { + const t = rebuildApp(); + const id = await inState(t, state); + for (const [ref, word] of [ + [internal.rebuild.interruptGeneration, "interrupt"], + [internal.rebuild.resumeGeneration, "resume"], + ] as const) + await refused( + t, + () => t.mutation(ref, { generationId: id, operator }), + `Generation 1 of documentTitle has no batch to ${word}`, + ); + }, +); + +// rebuild.sdp.md step3:123, fnResumeChain:131; write-pause.sdp.md fnGateAllows, restoreDoor. +test.each(["tenant", "restore"])( + "convex-test: a parked batch under %s records its error without checkpointing and resumes after an interrupt", + async (gate) => { + const t = rebuildApp(); + await create(t, "a"); + const id = await start(t); + if (gate === "tenant") + await t.mutation(internal.gate.closeGate, { + scopeKey: "tenant:tenant-a", + reason: "repair", + operator, + }); + else vi.stubEnv("MAINTENANCE_MODE", "restore"); + const before = await read(t, id); + const queued = await scheduled(t); + later(); + await batch(t, id); + expect(await read(t, id)).toStrictEqual({ + generation: before.generation, + progress: { + ...before.progress, + lastError: + gate === "tenant" + ? "write paused for tenant:tenant-a: repair" + : "write paused for all: restore", + updatedAt: 1010, + }, + }); + expect(await rows(t)).toEqual([]); + expect(await scheduled(t)).toStrictEqual(queued); + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator, + }); + if (gate === "tenant") + await t.mutation(internal.gate.resumeGate, { + scopeKey: "tenant:tenant-a", + operator, + }); + else vi.stubEnv("MAINTENANCE_MODE", "Restore"); + later(); + await t.mutation(internal.rebuild.resumeGeneration, { + generationId: id, + operator, + }); + const resumed = await read(t, id); + expect(resumed.progress).toStrictEqual({ + ...before.progress, + updatedAt: 1020, + }); + expect(resumed.generation.fence).toBe(3); + await batch(t, id); + expect(await rows(t)).toHaveLength(1); + }, +); + +// projection-contract.sdp.md step1:90, step3:92, step4:93, definition of older:64. +test.each(projectionCases)( + "convex-test: $mode / $role / $state / null=$isNull", + async ({ mode, role, state, isNull, expected }) => { + const t = rebuildApp(); + const prior = priorRow(state); + if (prior) await t.run((ctx) => ctx.db.insert("documentTitles", prior)); + const before = await rows(t); + const result = await t.run((ctx) => + applyProjection(ctx, readModel, { + tenantId: "tenant-a", + dto: { ...dto, title: isNull ? deletedTitle : "candidate" }, + versions: [version(3)], + mode, + generations: [{ role, generation: 1, projectionVersion: 2 }], + }), + ); + expect(result).toStrictEqual([expected]); + if (expected === "inserted" || expected === "updated") + expect(await rows(t)).toStrictEqual([ + { + ...candidateRow, + _id: before[0]?._id ?? expect.any(String), + _creationTime: before[0]?._creationTime ?? expect.any(Number), + }, + ]); + else if (expected === "deleted") expect(await rows(t)).toStrictEqual([]); + else expect(await rows(t)).toStrictEqual(before); + }, +); + +// projection-contract.sdp.md step3:92; generation-registry.sdp.md typeWritableGeneration; +// rebuild.sdp.md rule:69, rule:70, step5:125. +test.each(["building", "verifying", "verified"] as const)( + "convex-test: live commands create and update rows in %s with that state's role", + async (state) => { + const t = rebuildApp(); + await install(t, "documentSummary"); + await create(t, "old", "original", "tenant-a", "summary"); + const id = await start(t, { readModel: "documentSummary" }); + if (state !== "building") await batch(t, id); + if (state === "verified") await batch(t, id); + // A missing row after backfill stands for a subject that projected to none. + await t.run(async (ctx) => { + for (const r of await ctx.db.query("documentSummaries").collect()) + if (r.generation === 2) await ctx.db.delete(r._id); + }); + const firstUpdate = await amend(t, "old", "amended", true); + const oldRows = (await summaries(t)).filter((r) => r.key === "old"); + expect(oldRows.map((r) => r.generation).sort()).toEqual( + state === "building" ? [1] : [1, 2], + ); + for (const r of oldRows) + expect(r).toStrictEqual({ + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "tenant-a", + generation: r.generation, + key: "old", + projectionVersion: 1, + sourceVersions: firstUpdate.versions, + documentId: "old", + title: "amended", + status: "draft", + }); + const created = await create(t, "new", "new", "tenant-a", "summary"); + for (const r of (await summaries(t)).filter((r) => r.key === "new")) + expect(r.sourceVersions).toStrictEqual(created.versions); + expect( + (await summaries(t)) + .filter((r) => r.key === "new") + .map((r) => r.generation) + .sort(), + ).toEqual([1, 2]); + const updated = await amend(t, "new", "updated", true); + expect( + (await summaries(t)) + .filter((r) => r.key === "new") + .map((r) => ({ + generation: r.generation, + title: r.title, + sourceVersions: r.sourceVersions, + })), + ).toStrictEqual( + [1, 2].map((generation) => ({ + generation, + title: "updated", + sourceVersions: updated.versions, + })), + ); + expect((await read(t, id)).generation.state).toBe(state); + }, +); + +// native-harness.sdp.md fixtureReadModel:90; projection-contract.sdp.md:106; rebuild.online-rebuild-interrupt-resume.sdp.md:38. +test("convex-test: active version one and building version two project live writes and backfill by their own versions", async () => { + const t = rebuildApp(); + await install(t, "documentSummary"); + await create(t, "deleted-before", deletedTitle); + const activeId = await install(t, "documentTitle", 1); + expect((await rows(t)).find((r) => r.key === "deleted-before")).toStrictEqual( + { + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "tenant-a", + generation: 1, + projectionVersion: 1, + key: "deleted-before", + documentId: "deleted-before", + title: deletedTitle, + sourceVersions: [{ ...version(1), streamId: "deleted-before" }], + }, + ); + const id = await start(t, { projectionVersion: 2 }); + await create(t, "live", "live", "tenant-a", "twice"); + await create(t, "deleted-live", deletedTitle, "tenant-a", "twice"); + expect( + (await rows(t)) + .filter((r) => r.key === "live") + .map((r) => [r.generation, r.projectionVersion]), + ).toEqual([ + [1, 1], + [2, 2], + ]); + expect( + (await rows(t)) + .filter((r) => r.key === "deleted-live") + .map((r) => [r.generation, r.title]), + ).toEqual([[1, deletedTitle]]); + const active = await read(t, activeId); + await finish(t, id); + expect( + (await rows(t)).filter((r) => r.generation === 2).map((r) => r.key), + ).toEqual(["live"]); + expect(await read(t, activeId)).toStrictEqual(active); + expect( + (await rows(t)) + .filter((r) => r.generation === 1) + .map((r) => r.key) + .sort(), + ).toEqual(["deleted-before", "deleted-live", "live"]); +}); + +// rebuild.sdp.md verifySteps:127; projection-contract.sdp.md step1:90, step3:92. +test("convex-test: verify inserts a missing row, updates an older row and deletes a row projected to null", async () => { + const t = rebuildApp(); + for (const key of ["missing", "older", "deleted"]) + await create(t, key, key === "deleted" ? deletedTitle : key); + const id = await start(t, { projectionVersion: 2 }); + await batch(t, id); + await t.run(async (ctx) => { + for (const r of await ctx.db.query("documentTitles").collect()) { + if (r.key === "missing") await ctx.db.delete(r._id); + if (r.key === "older") + await ctx.db.patch(r._id, { + title: "old", + sourceVersions: [{ ...version(0), streamId: "older" }], + }); + } + await ctx.db.insert("documentTitles", { + tenantId: "tenant-a", + generation: 1, + projectionVersion: 2, + key: "deleted", + documentId: "deleted", + title: "old", + sourceVersions: [{ ...version(0), streamId: "deleted" }], + }); + }); + const before = await read(t, id); + const queue = await scheduled(t); + later(); + await batch(t, id); + expect(await read(t, id)).toStrictEqual({ + generation: { ...before.generation, state: "verified" }, + progress: { + ...before.progress, + pass: "idle", + cursor: null, + batchesDone: before.progress.batchesDone + 1, + misses: 3, + updatedAt: 1010, + }, + }); + expect(await scheduled(t)).toStrictEqual(queue); + expect( + (await rows(t)).sort((a, b) => a.key.localeCompare(b.key)), + ).toStrictEqual( + ["missing", "older"].map((key) => ({ + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "tenant-a", + generation: 1, + projectionVersion: 2, + key, + documentId: key, + title: key, + sourceVersions: [{ ...version(1), streamId: key }], + })), + ); +}); + +// projection-contract.sdp.md:106, step1:90, step3:92. +test.each(["active", "building", "verifying"] as const)( + "convex-test: a higher live update regains a null-projected row in role %s when the role permits", + async (role) => { + const t = rebuildApp(); + const apply = ( + title: string, + n: number, + mode: "live-created" | "live-updated" | "backfill", + ) => + t.run((ctx) => + applyProjection(ctx, documentTitle, { + tenantId: "tenant-a", + dto: { documentId: "document-a", title }, + versions: [version(n)], + mode, + generations: [{ role, generation: 1, projectionVersion: 2 }], + }), + ); + expect(await apply("original", 1, "live-created")).toEqual(["inserted"]); + expect(await apply(deletedTitle, 2, "backfill")).toEqual(["deleted"]); + expect(await rows(t)).toEqual([]); + expect(await apply("regained", 3, "live-updated")).toEqual([ + role === "building" ? "skipped-missing" : "inserted", + ]); + expect(await rows(t)).toStrictEqual( + role === "building" + ? [] + : [ + { + ...candidateRow, + title: "regained", + _id: expect.any(String), + _creationTime: expect.any(Number), + }, + ], + ); + }, +); + +// rebuild.sdp.md fnSwitchGeneration:133; generation-registry.sdp.md transitionCost. +test.each([undefined, 0, 2592000000])( + "convex-test: switch patches both generations atomically with rollback period %s", + async (rollbackPeriodMs) => { + const t = rebuildApp(); + const old = await install(t); + const id = await start(t); + await finish(t, id); + const previous = await read(t, old); + const next = await read(t, id); + later(); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator: " switch ", + ...(rollbackPeriodMs === undefined ? {} : { rollbackPeriodMs }), + }); + expect(await read(t, old)).toStrictEqual({ + generation: { + ...previous.generation, + state: "retired", + retiredAt: 1010, + retireAfter: 1010 + (rollbackPeriodMs ?? 604800000), + changedAt: 1010, + changedBy: "switch", + }, + progress: previous.progress, + }); + expect(await read(t, id)).toStrictEqual({ + generation: { + ...next.generation, + state: "active", + switchedAt: 1010, + changedAt: 1010, + changedBy: "switch", + }, + progress: next.progress, + }); + }, +); + +// rebuild.sdp.md fnSwitchGeneration:133. +test.each(states.filter((s) => s !== "verified"))( + "convex-test: switch refuses %s before an invalid rollback period", + async (state) => { + const t = rebuildApp(); + const id = await inState(t, state); + await refused( + t, + () => + t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator, + rollbackPeriodMs: -1, + }), + generationMessage(state, "switchGeneration", "verified"), + ); + }, +); + +// rebuild.sdp.md fnSwitchGeneration:133. +test.each([-1, 0.5, 2592000001, NaN, Infinity])( + "convex-test: switch refuses rollback period %s without changing either row", + async (rollbackPeriodMs) => { + const t = rebuildApp(); + await install(t); + const id = await start(t); + await finish(t, id); + await refused( + t, + () => + t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator, + rollbackPeriodMs, + }), + `The rollback period must be between 0 and 2592000000 ms, and ${rollbackPeriodMs} is not`, + ); + }, +); + +// rebuild.sdp.md fnSwitchGeneration:133, fnAbortGeneration:135. +test("convex-test: abort cannot remove the active generation while another awaits switch", async () => { + const t = rebuildApp(); + const active = await install(t); + const id = await start(t); + await finish(t, id); + await refused( + t, + () => + t.mutation(internal.rebuild.abortGeneration, { + generationId: active, + operator, + reason: "repair", + }), + generationMessage( + "active", + "abortGeneration", + "building, verifying or verified", + ), + ); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator, + }); + expect((await read(t, active)).generation.state).toBe("retired"); + expect((await read(t, id)).generation.state).toBe("active"); +}); + +// rebuild.sdp.md fnRollbackGeneration:134, verifySteps:127; rebuild.online-rebuild-interrupt-resume.sdp.md:38. +test("convex-test: rollback at version two repairs intervening creates, updates and a deletion before switch", async () => { + const t = rebuildApp(); + await install(t, "documentSummary"); + await create(t, "updated", "original"); + await create(t, "deleted", "original"); + const old = await start(t, { projectionVersion: 2 }); + await batch(t, old); + await t.run(async (ctx) => { + const r = await ctx.db + .query("documentTitles") + .withIndex("by_key", (q) => + q.eq("tenantId", "tenant-a").eq("generation", 1).eq("key", "updated"), + ) + .unique(); + await ctx.db.delete(r!._id); + }); + await finish(t, old); + expect((await read(t, old)).progress.misses).toBe(1); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: old, + operator, + }); + const id = await start(t, { projectionVersion: 2 }); + await finish(t, id); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator, + }); + const retiredRows = (await rows(t)).filter((r) => r.generation === 1); + await create(t, "created", "created", "tenant-a", "twice"); + const amended = await amend(t, "updated", "amended"); + await amend(t, "deleted", deletedTitle); + expect((await rows(t)).filter((r) => r.generation === 1)).toStrictEqual( + retiredRows, + ); + const before = await read(t, old); + const active = await read(t, id); + later(); + await t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator: " rollback ", + }); + const rolled = await read(t, old); + expect(rolled).toStrictEqual({ + generation: { + ...without(before.generation, "retiredAt", "retireAfter"), + state: "verifying", + fence: before.generation.fence + 1, + changedAt: 1010, + changedBy: "rollback", + }, + progress: { + ...without(before.progress, "lastError"), + pass: "verify", + cursor: null, + misses: 0, + updatedAt: 1010, + }, + }); + await lastSchedule(t, "verifyBatch", { + generationId: old, + fence: rolled.generation.fence, + }); + later(); + await batch(t, old); + const verified = await read(t, old); + expect(verified.generation).toStrictEqual({ + ...rolled.generation, + state: "verified", + }); + expect(verified.progress).toStrictEqual({ + ...rolled.progress, + pass: "idle", + batchesDone: rolled.progress.batchesDone + 1, + misses: 3, + updatedAt: 1020, + }); + expect(await read(t, id)).toStrictEqual(active); + expect( + (await rows(t)) + .filter((r) => r.generation === 1) + .sort((a, b) => a.key.localeCompare(b.key)), + ).toStrictEqual([ + { + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "tenant-a", + generation: 1, + projectionVersion: 2, + key: "created", + documentId: "created", + title: "created", + sourceVersions: [{ ...version(1), streamId: "created" }], + }, + { + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "tenant-a", + generation: 1, + projectionVersion: 2, + key: "updated", + documentId: "updated", + title: "amended", + sourceVersions: amended.versions, + }, + ]); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: old, + operator, + }); + expect((await read(t, old)).generation.state).toBe("active"); + expect((await read(t, id)).generation.state).toBe("retired"); +}); + +// rebuild.sdp.md fnRollbackGeneration:134, limitRollbackPeriod:150. +test.each(states.filter((s) => s !== "retired"))( + "convex-test: rollback refuses %s", + async (state) => { + const t = rebuildApp(); + const id = await inState(t, state); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: id, + operator, + }), + generationMessage(state, "rollbackGeneration", "retired"), + ); + }, +); + +// rebuild.sdp.md fnRollbackGeneration:134. The remaining five refusals are checked in their pinned order. +test("convex-test: rollback checks period, purge, pause, in-flight generation and projection in order", async () => { + const t = rebuildApp(); + const old = await install(t); + const active = await start(t); + await finish(t, active); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: active, + operator, + rollbackPeriodMs: 0, + }); + await patchGeneration( + t, + old, + { pauseRequired: true, projectionVersion: 99 }, + { pass: "purge" }, + ); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }), + "Generation 1 of documentTitle is past its rollback period", + ); + await patchGeneration(t, old, { retireAfter: Date.now() + 1 }); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }), + "Generation 1 of documentTitle is already being purged", + ); + await patchGeneration(t, old, {}, { pass: "idle" }); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }), + "Generation 1 of documentTitle needs a write pause and cannot be rolled back", + ); + await patchGeneration(t, old, { pauseRequired: false }); + const other = await start(t); + for (const state of ["building", "verifying", "verified"] as const) { + await patchGeneration(t, other, { state }); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }), + `Read model documentTitle already has generation 3 in ${state}`, + ); + } + await t.mutation(internal.rebuild.abortGeneration, { + generationId: other, + operator, + reason: "repair", + }); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }), + "Read model documentTitle declares no projection of version 99", + ); +}); + +// rebuild.sdp.md fnAbortGeneration:135. +test.each(["building", "verifying", "verified"] as const)( + "convex-test: abort from %s keeps its cursor and counts and ends the pass", + async (state) => { + const t = rebuildApp(); + await create(t, "a"); + await create(t, "b"); + const id = await start(t, { batchSize: 1 }); + await batch(t, id); + await patchGeneration( + t, + id, + { state }, + { misses: 4, rowsSkipped: 7, rowsPurged: 9 }, + ); + const before = await read(t, id); + const queue = await scheduled(t); + later(); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + reason: "é".repeat(128), + operator: " abort ", + }); + expect(await read(t, id)).toStrictEqual({ + generation: { + ...before.generation, + state: "aborted", + fence: before.generation.fence + 1, + changedAt: 1010, + changedBy: "abort", + }, + progress: { + ...before.progress, + pass: "idle", + lastError: `aborted: ${"é".repeat(128)}`, + updatedAt: 1010, + }, + }); + expect(await scheduled(t)).toStrictEqual(queue); + }, +); + +// rebuild.sdp.md fnAbortGeneration:135. +test.each(["", "x".repeat(257), "é".repeat(129)])( + "convex-test: abort refuses reason %j before checking state", + async (reason) => { + const t = rebuildApp(); + const id = await install(t); + await refused( + t, + () => + t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason, + }), + "The reason must be between 1 and 256 bytes", + ); + }, +); + +// rebuild.sdp.md fnAbortGeneration:135. +test.each(["active", "retired", "aborted", "purged"] as const)( + "convex-test: abort refuses %s", + async (state) => { + const t = rebuildApp(); + const id = await inState(t, state); + await refused( + t, + () => + t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "r", + }), + generationMessage( + state, + "abortGeneration", + "building, verifying or verified", + ), + ); + }, +); + +// rebuild.sdp.md fnRollbackGeneration:134, fnAbortGeneration:135. +test("convex-test: aborting a rollback leaves the serving generation active", async () => { + const t = rebuildApp(); + const old = await install(t); + const active = await start(t); + await finish(t, active); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: active, + operator, + }); + const serving = await read(t, active); + await t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }); + const before = await read(t, old); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: old, + operator, + reason: "stop rollback", + }); + expect(await read(t, active)).toStrictEqual(serving); + expect((await read(t, old)).generation).toStrictEqual({ + ...before.generation, + state: "aborted", + fence: before.generation.fence + 1, + }); + expect((await read(t, old)).progress).toStrictEqual({ + ...before.progress, + pass: "idle", + lastError: "aborted: stop rollback", + }); +}); + +// rebuild.sdp.md fnPurgeGeneration:136. +test.each(["building", "verifying", "verified", "active", "purged"] as const)( + "convex-test: purge refuses %s", + async (state) => { + const t = rebuildApp(); + const id = await inState(t, state); + await refused( + t, + () => + t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator, + }), + generationMessage(state, "purgeGeneration", "retired or aborted"), + ); + }, +); + +// rebuild.sdp.md fnPurgeGeneration:136, fnRollbackGeneration:134, limitRollbackPeriod:150. +test("convex-test: purge is refused before retireAfter and allowed at it while rollback is refused at it", async () => { + const t = rebuildApp(); + const old = await install(t); + const active = await start(t); + await finish(t, active); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: active, + operator, + rollbackPeriodMs: 10, + }); + await refused( + t, + () => + t.mutation(internal.rebuild.purgeGeneration, { + generationId: old, + operator, + }), + "Generation 1 of documentTitle is inside its rollback period", + ); + later(); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }), + "Generation 1 of documentTitle is past its rollback period", + ); + const before = await read(t, old); + await t.mutation(internal.rebuild.purgeGeneration, { + generationId: old, + operator: " purge ", + }); + expect(await read(t, old)).toStrictEqual({ + generation: { + ...before.generation, + fence: before.generation.fence + 1, + changedAt: 1010, + changedBy: "purge", + }, + progress: { + ...before.progress, + pass: "purge", + cursor: null, + updatedAt: 1010, + }, + }); + await lastSchedule(t, "purgeBatch", { + generationId: old, + fence: before.generation.fence + 1, + }); +}); + +async function purgeRows( + t: App, + generation: number, + tenantId: string, + count: number, +) { + await tenant(t, tenantId); + await t.run(async (ctx) => { + for (let i = 0; i < count; i++) { + const key = `document-${i}`; + await ctx.db.insert("documentTitles", { + tenantId, + generation, + key, + documentId: key, + title: key, + projectionVersion: 1, + sourceVersions: [{ ...version(1), tenantId, streamId: key }], + }); + } + }); +} + +// rebuild.sdp.md purgeSteps:128, fnPurgeGeneration:136; generation-registry.sdp.md tableGenerationProgress. +test("convex-test: purge crosses tenants and checkpoints only progress until the pass ends", async () => { + const t = rebuildApp(); + const id = await start(t, { batchSize: 2 }); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }); + await purgeRows(t, 1, "tenant-a", 3); + await purgeRows(t, 1, "tenant-z", 1); + later(); + await t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator, + }); + const before = await read(t, id); + expect(before.progress).not.toHaveProperty("lastError"); + later(); + await batch(t, id); + expect(await read(t, id)).toStrictEqual({ + generation: before.generation, + progress: { + ...before.progress, + cursor: { tenantId: "tenant-a", pageCursor: null }, + batchesDone: 1, + rowsPurged: 2, + updatedAt: 1020, + }, + }); + expect((await rows(t)).map((r) => r.tenantId).sort()).toEqual([ + "tenant-a", + "tenant-z", + ]); + later(); + await batch(t, id); + expect(await read(t, id)).toStrictEqual({ + generation: before.generation, + progress: { + ...before.progress, + cursor: { tenantId: "tenant-z", pageCursor: null }, + batchesDone: 2, + rowsPurged: 3, + updatedAt: 1030, + }, + }); + const queue = await scheduled(t); + later(); + await batch(t, id); + expect(await read(t, id)).toStrictEqual({ + generation: { ...before.generation, state: "purged" }, + progress: { + ...before.progress, + pass: "idle", + cursor: null, + batchesDone: 3, + rowsPurged: 4, + updatedAt: 1040, + }, + }); + expect(await rows(t)).toEqual([]); + expect(await scheduled(t)).toStrictEqual(queue); +}); + +// rebuild.sdp.md purgeSteps:128, limitPurgeBatch:147. The purge near its bound is an independent obligation. +test("convex-test: one more row than the purge cap takes exactly two batches on the tenant", async () => { + const t = rebuildApp(); + const id = await start(t, { batchSize: 2 }); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }); + await purgeRows(t, 1, "tenant-a", 3); + await t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator, + }); + const before = await read(t, id); + await batch(t, id); + expect(await rows(t)).toHaveLength(1); + expect(await read(t, id)).toStrictEqual({ + generation: before.generation, + progress: { + ...before.progress, + pass: "purge", + batchesDone: 1, + rowsPurged: 2, + cursor: { tenantId: "tenant-a", pageCursor: null }, + }, + }); + await batch(t, id); + expect(await rows(t)).toEqual([]); + expect(await read(t, id)).toStrictEqual({ + generation: { ...before.generation, state: "purged" }, + progress: { + ...before.progress, + pass: "idle", + batchesDone: 2, + rowsPurged: 3, + cursor: null, + }, + }); +}); + +// rebuild.sdp.md purgeSteps:128, limitPurgeBatch:147. A valid start caps batchSize below this limit; +// these stored progress rows exercise the defensive cap with a size above it. +test.each([256, 257])( + "convex-test: the read-model purge limit caps an oversized stored batch at 256 for %i rows", + async (count) => { + const t = rebuildApp(); + const id = await start(t); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }); + await patchGeneration(t, id, {}, { batchSize: 500 }); + await purgeRows(t, 1, "tenant-a", count); + await t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator, + }); + const before = await read(t, id); + later(); + await batch(t, id); + expect(await rows(t)).toHaveLength(count - 256); + expect(await read(t, id)).toStrictEqual({ + generation: before.generation, + progress: { + ...before.progress, + cursor: { tenantId: "tenant-a", pageCursor: null }, + rowsPurged: 256, + batchesDone: 1, + updatedAt: 1010, + }, + }); + later(); + await batch(t, id); + expect(await rows(t)).toEqual([]); + expect(await read(t, id)).toStrictEqual({ + generation: { ...before.generation, state: "purged" }, + progress: { + ...before.progress, + pass: "idle", + rowsPurged: count, + batchesDone: 2, + updatedAt: 1020, + }, + }); + }, +); + +// rebuild.sdp.md fnPurgeGeneration:136, fnInterruptGeneration:130, fnResumeGeneration:132, purgeSteps:128. +test.each(["retired", "aborted"] as const)( + "convex-test: an interrupted purge of %s resumes at its stored batch size", + async (state) => { + const t = rebuildApp(); + const id = await start(t, { batchSize: 2 }); + await finish(t, id); + if (state === "aborted") + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }); + else { + await t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator, + }); + const next = await start(t); + await finish(t, next); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: next, + operator, + rollbackPeriodMs: 0, + }); + } + await purgeRows(t, 1, "tenant-a", 5); + await t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator, + }); + await batch(t, id); + await refused( + t, + () => + t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator, + }), + "Generation 1 of documentTitle is already being purged; resumeGeneration continues it", + ); + const oldFence = (await read(t, id)).generation.fence; + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator, + }); + const interrupted = await read(t, id); + later(); + await t.mutation(internal.rebuild.resumeGeneration, { + generationId: id, + operator: " resume purge ", + }); + const resumed = await read(t, id); + expect(resumed).toStrictEqual({ + generation: { + ...interrupted.generation, + fence: oldFence + 2, + changedAt: 1010, + changedBy: "resume purge", + }, + progress: { + ...without(interrupted.progress, "lastError"), + updatedAt: 1010, + }, + }); + await lastSchedule(t, "purgeBatch", { + generationId: id, + fence: oldFence + 2, + }); + const beforeStale = await snapshot(t); + await t.mutation(internal.rebuild.purgeBatch, { + generationId: id, + fence: oldFence, + }); + expect(await snapshot(t)).toStrictEqual(beforeStale); + await batch(t, id); + expect(await rows(t)).toHaveLength(1); + expect((await read(t, id)).progress).toStrictEqual({ + ...resumed.progress, + rowsPurged: 4, + batchesDone: resumed.progress.batchesDone + 1, + }); + await batch(t, id); + const done = await read(t, id); + expect(done.generation).toStrictEqual({ + ...without(resumed.generation, "retiredAt", "retireAfter"), + state: "purged", + }); + expect(done.progress).toStrictEqual({ + ...resumed.progress, + pass: "idle", + cursor: null, + rowsPurged: 5, + batchesDone: resumed.progress.batchesDone + 2, + }); + }, +); + +// rebuild.sdp.md fnStartGeneration:115, fnFillTenants:139. +test("convex-test: a grant beside an empty tenant list refuses start with the fillTenants message", async () => { + const t = rebuildApp(); + await grantsWithoutTenants(t, 1); + await refused( + t, + () => start(t), + "The tenant list is empty while grants exist; fillTenants fills it from the grants table", + ); +}); + +// rebuild.sdp.md tableTenantFill:138, fnFillTenants:139, fillSteps:141, limitTenantFillBatch:142. +test.each([8, 9])( + "convex-test: fill walks %i distinct tenants with a bound of eight and a second fill inserts none", + async (count) => { + const t = rebuildApp(); + await grantsWithoutTenants(t, count); + expect(await tenants(t)).toEqual([]); + await t.mutation(internal.rebuild.fillTenants, { + operator: " fill operator ", + }); + const initial = await fill(t); + expect(initial).toStrictEqual({ + _id: expect.any(String), + _creationTime: expect.any(Number), + pass: "fill", + fence: 1, + cursor: null, + batchesDone: 0, + tenantsRead: 0, + tenantsInserted: 0, + startedAt: 1000, + startedBy: "fill operator", + changedAt: 1000, + changedBy: "fill operator", + updatedAt: 1000, + }); + await lastSchedule(t, "fillTenantsBatch", { fence: 1 }); + later(); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + expect(await fill(t)).toStrictEqual({ + ...initial, + cursor: "tenant-07", + batchesDone: 1, + tenantsRead: 8, + tenantsInserted: 8, + updatedAt: 1010, + }); + expect(await tenants(t)).toStrictEqual( + Array.from({ length: 8 }, (_, i) => ({ + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: `tenant-0${i}`, + createdAt: 1010, + })), + ); + await lastSchedule(t, "fillTenantsBatch", { fence: 1 }); + const queue = await scheduled(t); + later(); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + const done = { + ...initial, + pass: "idle", + cursor: null, + batchesDone: 2, + tenantsRead: count, + tenantsInserted: count, + updatedAt: 1020, + }; + expect(await fill(t)).toStrictEqual(done); + expect(await scheduled(t)).toStrictEqual(queue); + const inserted = await tenants(t); + expect(inserted).toHaveLength(count); + if (count === 9) + expect(inserted.at(-1)).toStrictEqual({ + _id: expect.any(String), + _creationTime: expect.any(Number), + tenantId: "tenant-08", + createdAt: 1020, + }); + later(); + await t.mutation(internal.rebuild.fillTenants, { operator: "again" }); + const reopened = { + ...initial, + fence: 2, + changedAt: 1030, + changedBy: "again", + updatedAt: 1030, + }; + expect(await fill(t)).toStrictEqual(reopened); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 2 }); + expect(await fill(t)).toStrictEqual({ + ...reopened, + cursor: "tenant-07", + tenantsRead: 8, + tenantsInserted: 0, + batchesDone: 1, + }); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 2 }); + expect(await fill(t)).toStrictEqual({ + ...reopened, + pass: "idle", + cursor: null, + tenantsRead: count, + tenantsInserted: 0, + batchesDone: 2, + }); + expect(await tenants(t)).toStrictEqual(inserted); + }, +); + +// rebuild.sdp.md fnFillTenants:139, fnStartGeneration:115. +test.each(["building", "verifying", "verified"] as const)( + "convex-test: fillTenants refuses a generation in %s", + async (state) => { + const t = rebuildApp(); + const id = await start(t, { readModel: "documentSummary" }); + await patchGeneration(t, id, { state }); + await refused( + t, + () => t.mutation(internal.rebuild.fillTenants, { operator }), + `Read model documentSummary has generation 1 in ${state}; fillTenants needs no generation in flight`, + ); + }, +); + +// rebuild.sdp.md fnStartGeneration:115, fnFillTenants:139, fillSteps:141. +test("convex-test: start refuses an unfinished fill even after the tenant list has rows", async () => { + const t = rebuildApp(); + await grantsWithoutTenants(t, 9); + await t.mutation(internal.rebuild.fillTenants, { operator }); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + await refused( + t, + () => start(t), + "The tenant list is being filled; startGeneration waits until fillTenants has ended", + ); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + const id = await start(t); + expect((await read(t, id)).generation.state).toBe("building"); +}); + +// rebuild.sdp.md fillSteps:141, fnFillTenants:139; write-pause.sdp.md fnGateAllows. +test("convex-test: fill ignores a tenant gate, parks under the deployment gate and keeps its cursor on resumption", async () => { + const t = rebuildApp(); + await grantsWithoutTenants(t, 9); + await t.mutation(internal.gate.closeGate, { + scopeKey: "tenant:tenant-00", + reason: "tenant repair", + operator, + }); + await t.mutation(internal.rebuild.fillTenants, { operator }); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + expect((await fill(t))?.tenantsInserted).toBe(8); + await t.mutation(internal.gate.closeGate, { + scopeKey: "all", + reason: "repair", + operator, + }); + const before = await fill(t); + const queue = await scheduled(t); + const listed = await tenants(t); + later(); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + expect(await fill(t)).toStrictEqual({ + ...before, + lastError: "write paused for all: repair", + updatedAt: 1010, + }); + expect(await scheduled(t)).toStrictEqual(queue); + expect(await tenants(t)).toStrictEqual(listed); + await t.mutation(internal.gate.resumeGate, { scopeKey: "all", operator }); + later(); + await t.mutation(internal.rebuild.fillTenants, { operator: "continued" }); + expect(await fill(t)).toStrictEqual({ + ...before, + fence: 2, + changedAt: 1020, + changedBy: "continued", + updatedAt: 1020, + }); + await lastSchedule(t, "fillTenantsBatch", { fence: 2 }); + const current = await snapshot(t); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + expect(await snapshot(t)).toStrictEqual(current); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 2 }); + expect(await fill(t)).toStrictEqual({ + ...before, + pass: "idle", + cursor: null, + fence: 2, + changedAt: 1020, + changedBy: "continued", + updatedAt: 1020, + batchesDone: 2, + tenantsRead: 9, + tenantsInserted: 9, + }); +}); + +// rebuild.sdp.md fnFillTenants:139, fillSteps:141. +test("convex-test: fill refuses exactly the restore door and its batch parks there", async () => { + const t = rebuildApp(); + await grantsWithoutTenants(t, 1); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + await refused( + t, + () => t.mutation(internal.rebuild.fillTenants, { operator }), + "fillTenants is refused while the restore door is closed", + ); + vi.stubEnv("MAINTENANCE_MODE", "Restore"); + await t.mutation(internal.rebuild.fillTenants, { operator }); + const before = await fill(t); + const queue = await scheduled(t); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + later(); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + expect(await fill(t)).toStrictEqual({ + ...before, + lastError: "write paused for all: restore", + updatedAt: 1010, + }); + expect(await tenants(t)).toEqual([]); + expect(await scheduled(t)).toStrictEqual(queue); +}); + +// rebuild.sdp.md fillSteps:141. +test("convex-test: fill batches with no row, a stale fence or an idle pass write nothing", async () => { + const t = rebuildApp(); + const empty = await snapshot(t); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + expect(await snapshot(t)).toStrictEqual(empty); + await t.mutation(internal.rebuild.fillTenants, { operator }); + const filling = await snapshot(t); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 0 }); + expect(await snapshot(t)).toStrictEqual(filling); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + const idle = await snapshot(t); + await t.mutation(internal.rebuild.fillTenantsBatch, { fence: 1 }); + expect(await snapshot(t)).toStrictEqual(idle); +}); + +// rebuild.sdp.md fnGetGenerations:137, limitGenerationsListed:151. +test("convex-test: getGenerations returns the newest twenty complete row pairs from twenty-one generations", async () => { + const t = rebuildApp(); + const ids: GenerationId[] = []; + for (let i = 0; i < 21; i++) { + const id = await start(t); + ids.push(id); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }); + } + const before = await snapshot(t); + expect( + await t.query(internal.rebuild.getGenerations, { + readModel: "documentTitle", + }), + ).toStrictEqual( + await Promise.all( + ids + .slice(1) + .reverse() + .map((id) => read(t, id)), + ), + ); + expect( + await t.query(internal.rebuild.getGenerations, { readModel: "unknown" }), + ).toEqual([]); + expect(await snapshot(t)).toStrictEqual(before); +}); + +// actor-and-scope.sdp.md fnAssertOperator:91; rebuild.sdp.md fnStartGeneration:115 through fnFillTenants:139. +test("convex-test: every operator entry accepts and records the trimmed 512-byte bound", async () => { + const t = rebuildApp(); + const atBound = "é".repeat(256); + const stated = ` ${atBound} `; + const id = await start(t, { operator: stated }); + expect((await read(t, id)).generation.startedBy).toBe(atBound); + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator: stated, + }); + expect((await read(t, id)).generation.changedBy).toBe(atBound); + await t.mutation(internal.rebuild.resumeGeneration, { + generationId: id, + operator: stated, + }); + expect((await read(t, id)).generation.changedBy).toBe(atBound); + await finish(t, id); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator: stated, + }); + expect((await read(t, id)).generation.changedBy).toBe(atBound); + const next = await start(t); + await finish(t, next); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: next, + operator, + }); + await t.mutation(internal.rebuild.rollbackGeneration, { + generationId: id, + operator: stated, + }); + expect((await read(t, id)).generation.changedBy).toBe(atBound); + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator: stated, + reason: "r", + }); + expect((await read(t, id)).generation.changedBy).toBe(atBound); + await t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator: stated, + }); + expect((await read(t, id)).generation.changedBy).toBe(atBound); + await t.mutation(internal.rebuild.fillTenants, { operator: stated }); + expect((await fill(t))?.startedBy).toBe(atBound); + expect((await fill(t))?.changedBy).toBe(atBound); +}); + +// rebuild.sdp.md convexSurface:112, fnStartGeneration:115, fnBackfillBatch:117, fnVerifyBatch:118, fnSwitchGeneration:133. +test("convex-test: the example composition installs orderSummary through its registered rebuild entries", async () => { + const t = productionTest(); + const id = await t.mutation(exampleInternal.rebuild.startGeneration, { + readModel: "orderSummary", + operator, + }); + await t.mutation(exampleInternal.rebuild.backfillBatch, { + generationId: id, + fence: 1, + }); + await t.mutation(exampleInternal.rebuild.verifyBatch, { + generationId: id, + fence: 1, + }); + await t.mutation(exampleInternal.rebuild.switchGeneration, { + generationId: id, + operator, + }); + const listed = await t.query(exampleInternal.rebuild.getGenerations, { + readModel: "orderSummary", + }); + expect(listed).toHaveLength(1); + expect(listed[0]?.generation).toStrictEqual({ + _id: id, + _creationTime: expect.any(Number), + readModel: "orderSummary", + generation: 1, + projectionVersion: 1, + state: "active", + pauseRequired: false, + fence: 1, + startedAt: 1000, + startedBy: operator, + changedAt: 1000, + changedBy: operator, + switchedAt: 1000, + }); + expect(listed[0]?.progress).toStrictEqual({ + _id: expect.any(String), + _creationTime: expect.any(Number), + generationId: id, + pass: "idle", + batchSize: 100, + cursor: null, + ...zeros, + batchesDone: 2, + updatedAt: 1000, + }); +}); + +// rebuild.sdp.md step1:121, verifySteps:127, purgeSteps:128. +test.each([ + ["backfillBatch", "active", "backfill"], + ["backfillBatch", "building", "verify"], + ["verifyBatch", "building", "verify"], + ["verifyBatch", "verifying", "backfill"], + ["purgeBatch", "active", "purge"], + ["purgeBatch", "retired", "idle"], + ["purgeBatch", "aborted", "idle"], +] as const)( + "convex-test: %s leaves state %s and pass %s untouched", + async (entry, state, pass) => { + const t = rebuildApp(); + const id = await inState(t, state, pass); + await purgeRows(t, 1, "tenant-a", 1); + const before = await snapshot(t); + await t.mutation(internal.rebuild[entry], { generationId: id, fence: 1 }); + expect(await snapshot(t)).toStrictEqual(before); + }, +); + +// rebuild.sdp.md fnResumeChain:131, fnResumeGeneration:132, verifySteps:127. +test("convex-test: resuming verify preserves its counts and cursor and schedules verify with the new fence", async () => { + const t = rebuildApp(); + for (let i = 0; i < 3; i++) await create(t, `document-${i}`); + const id = await start(t, { batchSize: 1 }); + for ( + let i = 0; + i < 5 && (await read(t, id)).progress.pass === "backfill"; + i++ + ) + await batch(t, id); + expect((await read(t, id)).generation.state).toBe("verifying"); + await t.run(async (ctx) => { + const row = await ctx.db + .query("documentTitles") + .withIndex("by_key", (q) => + q + .eq("tenantId", "tenant-a") + .eq("generation", 1) + .eq("key", "document-0"), + ) + .unique(); + await ctx.db.delete(row!._id); + }); + await batch(t, id); + const before = await read(t, id); + expect(before.progress.misses).toBe(1); + expect(before.progress.cursor).not.toBeNull(); + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator, + }); + later(); + await t.mutation(internal.rebuild.resumeGeneration, { + generationId: id, + operator: "resume verify", + }); + const resumed = await read(t, id); + expect(resumed).toStrictEqual({ + generation: { + ...before.generation, + fence: 3, + changedAt: 1010, + changedBy: "resume verify", + }, + progress: { ...before.progress, updatedAt: 1010 }, + }); + await lastSchedule(t, "verifyBatch", { generationId: id, fence: 3 }); + await batch(t, id); + expect((await read(t, id)).generation).toStrictEqual(resumed.generation); + expect((await read(t, id)).progress.cursor).not.toStrictEqual( + resumed.progress.cursor, + ); + expect((await read(t, id)).progress).toStrictEqual({ + ...resumed.progress, + cursor: { tenantId: "tenant-a", pageCursor: expect.any(String) }, + rowsSkipped: resumed.progress.rowsSkipped + 1, + batchesDone: resumed.progress.batchesDone + 1, + }); +}); + +// rebuild.sdp.md step3:123, verifySteps:127, purgeSteps:128; write-pause.sdp.md fnGateAllows:66. +test.each(["verify", "purge"] as const)( + "convex-test: %s also parks under a closed tenant without touching rows or checkpoint counts", + async (pass) => { + const t = rebuildApp(); + await create(t, "a"); + const id = await start(t); + await batch(t, id); + if (pass === "purge") { + await t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }); + await t.mutation(internal.rebuild.purgeGeneration, { + generationId: id, + operator, + }); + } + await t.mutation(internal.gate.closeGate, { + scopeKey: "tenant:tenant-a", + reason: "repair", + operator, + }); + const before = await read(t, id); + const written = await rows(t); + const queue = await scheduled(t); + later(); + await batch(t, id); + expect(await read(t, id)).toStrictEqual({ + generation: before.generation, + progress: { + ...before.progress, + lastError: "write paused for tenant:tenant-a: repair", + updatedAt: 1010, + }, + }); + expect(await rows(t)).toStrictEqual(written); + expect(await scheduled(t)).toStrictEqual(queue); + }, +); + +// rebuild.sdp.md fnStartGeneration:115, fnSwitchGeneration:133; generation-registry.sdp.md transitionCost. +test("convex-test: a failed activation commits neither half of the switch", async () => { + const t = rebuildApp(true); + const old = await install(t); + const id = await start(t); + await finish(t, id); + const before = await snapshot(t); + await expect( + t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator, + }), + ).rejects.toThrow(); + expect(await snapshot(t)).toStrictEqual(before); + expect((await read(t, old)).generation.state).toBe("active"); + expect((await read(t, id)).generation.state).toBe("verified"); +}); diff --git a/tests/types/rebuild.test-d.ts b/tests/types/rebuild.test-d.ts new file mode 100644 index 0000000..655bd4d --- /dev/null +++ b/tests/types/rebuild.test-d.ts @@ -0,0 +1,993 @@ +import type { + FunctionArgs, + FunctionReturnType, + FunctionReference, + PaginationResult, +} from "convex/server"; +import { expectTypeOf, test } from "vitest"; +import { internal as example } from "../../example/convex/_generated/api.js"; +import { internal as fixture } from "../../fixture/convex/_generated/api.js"; +import type { Id } from "../../fixture/convex/_generated/dataModel.js"; +import type { MutationCtx } from "../../fixture/convex/_generated/server.js"; +import type { ListArgs } from "../../src/context/index.js"; +import type { Projection, ReadModel } from "../../src/read-model/projection.js"; +import { + batchSizeFor, + rebuildBatchCeiling, + limitPurgeBatch, + resumeChain, + type RebuildConfig, + type RebuildTarget, + type BatchRef, + type FillBatchRef, +} from "../../src/read-model/rebuild.js"; + +type Generation = { + _id: Id<"generations">; + _creationTime: number; + readModel: string; + generation: number; + projectionVersion: number; + state: + | "building" + | "verifying" + | "verified" + | "active" + | "retired" + | "aborted" + | "purged"; + pauseRequired: boolean; + fence: number; + startedAt: number; + startedBy: string; + changedAt: number; + changedBy: string; + switchedAt?: number; + retiredAt?: number; + retireAfter?: number; +}; +type Progress = { + _id: Id<"generationProgress">; + _creationTime: number; + generationId: Id<"generations">; + pass: "backfill" | "verify" | "purge" | "idle"; + batchSize: number; + cursor: { + tenantId: string; + pageCursor: string | null; + streamId?: string; + eventCursor?: string | null; + } | null; + batchesDone: number; + rowsWritten: number; + rowsSkipped: number; + misses: number; + rowsPurged: number; + lastError?: string; + updatedAt: number; +}; + +// rebuild.sdp.md fnStartGeneration:115; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example startGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ + readModel: string; + projectionVersion?: number; + batchSize?: number; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf>(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { + readModel: string; + projectionVersion?: number; + batchSize?: number; + operator: string; + }, + Id<"generations"> + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.startGeneration, { + readModel: "documentTitle", + operator: "operator", + projectionVersion: 2, + batchSize: 1, + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.startGeneration, { + readModel: "documentTitle", + operator: "operator", + projectionVersion: "2", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnInterruptGeneration:130; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example interruptGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.interruptGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.interruptGeneration, { generationId }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnResumeGeneration:132; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example resumeGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.resumeGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.resumeGeneration, { + generationId, + operator: "operator", + batchSize: 2, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnSwitchGeneration:133; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example switchGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ + generationId: Id<"generations">; + rollbackPeriodMs?: number; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { + generationId: Id<"generations">; + rollbackPeriodMs?: number; + operator: string; + }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.switchGeneration, { + generationId, + operator: "operator", + rollbackPeriodMs: 0, + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.switchGeneration, { + generationId, + operator: "operator", + rollbackPeriodMs: "0", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnRollbackGeneration:134; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example rollbackGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.rollbackGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.rollbackGeneration, { + generationId, + operator: 1, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnAbortGeneration:135; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example abortGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ + generationId: Id<"generations">; + reason: string; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; reason: string; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.abortGeneration, { + generationId, + operator: "operator", + reason: "repair", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.abortGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnPurgeGeneration:136; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example purgeGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.purgeGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.purgeGeneration, { + generationId, + operator: "operator", + batchSize: 1, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnBackfillBatch:117; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example backfillBatch arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; fence: number }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; fence: number }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.backfillBatch, { + generationId, + fence: 1, + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.backfillBatch, { + generationId, + fence: "1", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnVerifyBatch:118; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example verifyBatch arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; fence: number }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; fence: number }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.verifyBatch, { generationId, fence: 1 }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.verifyBatch, { generationId }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnPurgeBatch:119; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example purgeBatch arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; fence: number }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; fence: number }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.purgeBatch, { generationId, fence: 1 }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.purgeBatch, { + generationId, + fence: 1, + batchSize: 1, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnFillTenants:139; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example fillTenants arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference<"mutation", "internal", { operator: string }, null> + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.fillTenants, { operator: "operator" }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.fillTenants, {}); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnFillTenantsBatch:140; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example fillTenantsBatch arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ fence: number }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference<"mutation", "internal", { fence: number }, null> + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(example.rebuild.fillTenantsBatch, { fence: 1 }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(example.rebuild.fillTenantsBatch, { + fence: 1, + generationId, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnGetGenerations:137; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: example getGenerations arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ readModel: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf<{ generation: Generation; progress: Progress }[]>(); + expectTypeOf().toExtend< + FunctionReference< + "query", + "internal", + { readModel: string }, + { generation: Generation; progress: Progress }[] + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runQuery(example.rebuild.getGenerations, { + readModel: "documentTitle", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runQuery(example.rebuild.getGenerations, { + readModel: "documentTitle", + operator: "operator", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnStartGeneration:115; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture startGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ + readModel: string; + projectionVersion?: number; + batchSize?: number; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf>(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { + readModel: string; + projectionVersion?: number; + batchSize?: number; + operator: string; + }, + Id<"generations"> + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.startGeneration, { + readModel: "documentTitle", + operator: "operator", + projectionVersion: 2, + batchSize: 1, + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.startGeneration, { + readModel: "documentTitle", + operator: "operator", + projectionVersion: "2", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnInterruptGeneration:130; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture interruptGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.interruptGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.interruptGeneration, { generationId }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnResumeGeneration:132; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture resumeGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.resumeGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.resumeGeneration, { + generationId, + operator: "operator", + batchSize: 2, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnSwitchGeneration:133; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture switchGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ + generationId: Id<"generations">; + rollbackPeriodMs?: number; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { + generationId: Id<"generations">; + rollbackPeriodMs?: number; + operator: string; + }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.switchGeneration, { + generationId, + operator: "operator", + rollbackPeriodMs: 0, + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.switchGeneration, { + generationId, + operator: "operator", + rollbackPeriodMs: "0", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnRollbackGeneration:134; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture rollbackGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.rollbackGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.rollbackGeneration, { + generationId, + operator: 1, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnAbortGeneration:135; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture abortGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ + generationId: Id<"generations">; + reason: string; + operator: string; + }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; reason: string; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.abortGeneration, { + generationId, + operator: "operator", + reason: "repair", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.abortGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnPurgeGeneration:136; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture purgeGeneration arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; operator: string }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.purgeGeneration, { + generationId, + operator: "operator", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.purgeGeneration, { + generationId, + operator: "operator", + batchSize: 1, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnBackfillBatch:117; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture backfillBatch arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; fence: number }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; fence: number }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.backfillBatch, { + generationId, + fence: 1, + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.backfillBatch, { + generationId, + fence: "1", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnVerifyBatch:118; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture verifyBatch arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; fence: number }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; fence: number }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.verifyBatch, { generationId, fence: 1 }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.verifyBatch, { generationId }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnPurgeBatch:119; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture purgeBatch arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ generationId: Id<"generations">; fence: number }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; fence: number }, + null + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.purgeBatch, { generationId, fence: 1 }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.purgeBatch, { + generationId, + fence: 1, + batchSize: 1, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnFillTenants:139; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture fillTenants arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ operator: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference<"mutation", "internal", { operator: string }, null> + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.fillTenants, { operator: "operator" }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.fillTenants, {}); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnFillTenantsBatch:140; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture fillTenantsBatch arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ fence: number }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf(); + expectTypeOf().toExtend< + FunctionReference<"mutation", "internal", { fence: number }, null> + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runMutation(fixture.rebuild.fillTenantsBatch, { fence: 1 }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runMutation(fixture.rebuild.fillTenantsBatch, { + fence: 1, + generationId, + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = + true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md fnGetGenerations:137; generation-registry.sdp.md tableGenerations, tableGenerationProgress. +test("compiled: fixture getGenerations arguments, visibility and return", () => { + expectTypeOf< + FunctionArgs + >().toEqualTypeOf<{ readModel: string }>(); + expectTypeOf< + FunctionReturnType + >().toEqualTypeOf<{ generation: Generation; progress: Progress }[]>(); + expectTypeOf().toExtend< + FunctionReference< + "query", + "internal", + { readModel: string }, + { generation: Generation; progress: Progress }[] + > + >(); + const ctx = {} as MutationCtx; + const generationId = "generation" as Id<"generations">; + void ctx.runQuery(fixture.rebuild.getGenerations, { + readModel: "documentTitle", + }); + // @ts-expect-error The entry accepts only the arguments its validator declares. + void ctx.runQuery(fixture.rebuild.getGenerations, { + readModel: "documentTitle", + operator: "operator", + }); + // @ts-expect-error The return validator does not accept a boolean. + const wrong: FunctionReturnType = true; + void [generationId, wrong]; +}); + +// rebuild.sdp.md typeRebuildTarget:113, typeRebuildConfig:114, fnBatchSizeFor:116, fnResumeChain:131, limitPurgeBatch:147. +test("compiled: rebuild configuration and helpers have the pinned signatures", () => { + type Dto = { documentId: string }; + type Row = { documentId: string }; + expectTypeOf>().toEqualTypeOf<{ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + readModel: ReadModel; + source: { contextId: string; streamType: string }; + list: FunctionReference< + "query", + "internal", + ListArgs, + PaginationResult + >; + }>(); + expectTypeOf().toEqualTypeOf< + FunctionReference< + "mutation", + "internal", + { generationId: Id<"generations">; fence: number }, + null + > + >(); + expectTypeOf().toEqualTypeOf< + FunctionReference<"mutation", "internal", { fence: number }, null> + >(); + expectTypeOf().toEqualTypeOf<{ + targets: readonly RebuildTarget[]; + refs: { + backfillBatch: BatchRef; + verifyBatch: BatchRef; + purgeBatch: BatchRef; + fillTenantsBatch: FillBatchRef; + }; + }>(); + type AnyModel = RebuildTarget["readModel"]; + expectTypeOf().parameters.toEqualTypeOf< + [readModel: AnyModel, requested?: number] + >(); + expectTypeOf().parameters.toEqualTypeOf< + [readModel: AnyModel] + >(); + expectTypeOf().parameters.toEqualTypeOf< + [readModel: AnyModel] + >(); + const model = {} as ReadModel; + expectTypeOf(batchSizeFor(model)).toEqualTypeOf(); + expectTypeOf(batchSizeFor(model, 1)).toEqualTypeOf(); + expectTypeOf(rebuildBatchCeiling(model)).toEqualTypeOf(); + expectTypeOf(limitPurgeBatch(model)).toEqualTypeOf(); + // @ts-expect-error The purge limit is a function of a read model, not a numeric budget. + limitPurgeBatch(16384); + // @ts-expect-error A requested batch size is numeric. + batchSizeFor(model, "1"); + const ctx = {} as MutationCtx; + const config = {} as RebuildConfig; + const id = "generation" as Id<"generations">; + expectTypeOf(resumeChain(ctx, config, id, "operator")).toEqualTypeOf< + Promise + >(); + // @ts-expect-error Resumption keeps the progress row's batch size. + void resumeChain(ctx, config, id, "operator", 1); +}); + +// projection-contract.sdp.md typeReadModel:67, typeProjection:66. +test("compiled: ReadModel declares a nonempty readonly list of projections", () => { + type Dto = { documentId: string }; + type Row = { documentId: string }; + expectTypeOf["projections"]>().toEqualTypeOf< + readonly [Projection, ...Projection[]] + >(); + // @ts-expect-error Every read model declares at least one projection. + const empty: ReadModel["projections"] = []; + // @ts-expect-error The old singular projection field is not part of ReadModel. + type Singular = ReadModel["projection"]; + void empty; + void ({} as Singular); +}); From 964efbbf18c912063839c6238378e1c5a9d3d37e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:08:32 +0200 Subject: [PATCH 080/155] build: 3b rebuild proof, scheduling of unfinished passes and the fixture's two versions Co-Authored-By: Claude Opus 5.5 --- tests/pure/rebuild.test.ts | 26 +++++++++++++++++++++++++- tests/simulator/rebuild.test.ts | 10 ++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/pure/rebuild.test.ts b/tests/pure/rebuild.test.ts index a2c9b5b..b062e76 100644 --- a/tests/pure/rebuild.test.ts +++ b/tests/pure/rebuild.test.ts @@ -1,5 +1,9 @@ import { expect, test } from "vitest"; -import { documentTitle } from "../../fixture/convex/documentTitles.js"; +import { deletedTitle } from "../../fixture/convex/depot/streams.js"; +import { + documentTitle, + projectionFaultTitle, +} from "../../fixture/convex/documentTitles.js"; import { applyProjection, projectionOf, @@ -25,6 +29,26 @@ test("pure: projectionOf selects the declared version and the latest projection ); }); +// native-harness.sdp.md fixtureReadModel:90: version 2 projects the deleted document to null, +// version 1 projects it as any other, and both throw for the fault title. +test("pure: the fixture's documentTitle versions differ only on the deleted document", () => { + const [second, first] = documentTitle.projections; + const live = { documentId: "d", title: "live" }; + const deleted = { documentId: "d", title: deletedTitle }; + expect(second.project("tenant-a", live, [])).toStrictEqual(live); + expect(first?.project("tenant-a", live, [])).toStrictEqual(live); + expect(second.project("tenant-a", deleted, [])).toBeNull(); + expect(first?.project("tenant-a", deleted, [])).toStrictEqual(deleted); + for (const projection of [second, first]) + expect(() => + projection?.project( + "tenant-a", + { documentId: "d", title: projectionFaultTitle }, + [], + ), + ).toThrow(); +}); + // rebuild.sdp.md fnBatchSizeFor:116, limitBatchSize:146, limitPurgeBatch:147. test.each([ [16384, 100, 256], diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts index 7a25a4e..957ff11 100644 --- a/tests/simulator/rebuild.test.ts +++ b/tests/simulator/rebuild.test.ts @@ -419,12 +419,15 @@ test("convex-test: page cursors continue, tenants cross in tenant-ID order and v updatedAt: 1010, }, }); + // step6: a pass that is not done schedules the same batch with the same fence. + await lastSchedule(t, "backfillBatch", { generationId: id, fence: 1 }); later(); await batch(t, id); expect((await read(t, id)).progress.cursor).toStrictEqual({ tenantId: "tenant-z", pageCursor: null, }); + await lastSchedule(t, "backfillBatch", { generationId: id, fence: 1 }); expect((await rows(t)).map((r) => r.key).sort()).toEqual(["a1", "a2", "a3"]); later(); await batch(t, id); @@ -448,6 +451,8 @@ test("convex-test: page cursors continue, tenants cross in tenant-ID order and v pageCursor: first.continueCursor, }); expect((await read(t, id)).progress.rowsSkipped).toBe(2); + // verifySteps: a verify pass that is not done schedules verifyBatch with the same fence. + await lastSchedule(t, "verifyBatch", { generationId: id, fence: 1 }); }); // rebuild.sdp.md step2:122, step6:126. The empty tenant is an independent tenant-list obligation. @@ -1490,6 +1495,11 @@ test("convex-test: purge crosses tenants and checkpoints only progress until the "tenant-a", "tenant-z", ]); + // purgeSteps: a purge pass that is not done schedules purgeBatch with the same fence. + await lastSchedule(t, "purgeBatch", { + generationId: id, + fence: before.generation.fence, + }); later(); await batch(t, id); expect(await read(t, id)).toStrictEqual({ From bce1239ea3b6453c8b49c970a27db009e2c2d3f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:12:31 +0200 Subject: [PATCH 081/155] build: 3b the setups install each read model through its first rebuild Co-Authored-By: Claude Opus 5.5 --- ...nd-pipeline.failure-before-receipt.test.ts | 6 +- ...kfill.probe-6-context-batch-cursor.test.ts | 6 +- ...ckfill.probe-6-transaction-failure.test.ts | 6 +- tests/native/first-experiment-steps.ts | 11 +- ...cceptance-production-configuration.test.ts | 3 +- tests/native/gate.test.ts | 6 +- ...y.command-without-generation-fails.test.ts | 6 +- tests/native/migration-state.ts | 7 +- tests/native/migration-steps.ts | 26 ++- tests/native/order-use-case.ts | 18 +-- tests/native/production-orders.test.ts | 14 +- tests/native/read-models.test.ts | 47 ++++-- tests/native/rebuild-install.ts | 114 +++++++++++++ tests/native/scheduler-cases.ts | 6 +- tests/pure/read-model.test.ts | 5 +- tests/simulator/example.test.ts | 30 ++-- tests/simulator/fixture-faults.test.ts | 6 +- tests/simulator/gate-pipeline.test.ts | 6 +- tests/simulator/gate-support.ts | 62 +++++-- tests/simulator/read-model.test.ts | 153 +++++++++++------- 20 files changed, 378 insertions(+), 160 deletions(-) create mode 100644 tests/native/rebuild-install.ts diff --git a/tests/native/command-pipeline.failure-before-receipt.test.ts b/tests/native/command-pipeline.failure-before-receipt.test.ts index a65046a..33ac051 100644 --- a/tests/native/command-pipeline.failure-before-receipt.test.ts +++ b/tests/native/command-pipeline.failure-before-receipt.test.ts @@ -1,3 +1,4 @@ +import { installFixtureReadModel } from "./rebuild-install.js"; import { ref, specTest, @@ -79,10 +80,7 @@ async function readModelTables(backend: Backend) { test("native: a projection that throws after step 9 wrote the first read model's row leaves no read-model row, no receipt and no event or state change", async () => { const backend = await fixtureBackend(); for (const readModel of ["documentSummary", "documentTitle"]) - await backend.admin.run(getFunctionName(internal.readModels.activate), { - readModel, - startedBy: { kind: "operator", id: "native-test" }, - }); + await installFixtureReadModel(backend, readModel); await backend.admin.run(getFunctionName(internal.grants.grant), { tenantId: "t-1", principalKind: "human", diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.test.ts index 70cd967..265bd96 100644 --- a/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.test.ts +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-context-batch-cursor.test.ts @@ -34,7 +34,11 @@ async function checkpoint(backend: Backend) { const generation = (await backend.admin.readTable("generations")).find( (row) => row.generation === 2, )!; - expect((generation.cursor as { pageCursor: string }).pageCursor).toBe( + const progress = (await backend.admin.readTable("generationProgress")).find( + (row) => row.generationId === generation._id, + )!; + // generation-registry.sdp.md:74: the parent checkpoint cursor is on generationProgress. + expect((progress.cursor as { pageCursor: string }).pageCursor).toBe( component.cursor, ); return { component, generation }; diff --git a/tests/native/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.test.ts b/tests/native/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.test.ts index e471ffc..be87c14 100644 --- a/tests/native/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.test.ts +++ b/tests/native/f17-migrations-fits-generation-backfill.probe-6-transaction-failure.test.ts @@ -75,8 +75,10 @@ bindExample(contract, (): World => ({ rows: 0, batch: 0 }), { expect((failed.worker!.args as { cursor: string }[])[0]!.cursor).toBe( failed.component.cursor, ); - expect(failed.generation.batchesDone).toBe(1); - expect(failed.generation.rowsWritten).toBe(world.batch); + // generation-registry.sdp.md:74: failed batches leave the progress checkpoint unchanged. + expect(failed.progress.batchesDone).toBe(1); + // generation-registry.sdp.md:74: only the committed batch contributed rowsWritten. + expect(failed.progress.rowsWritten).toBe(world.batch); expect(failed.rows.map((row) => row.key).sort()).toEqual([ "doc-00", "doc-01", diff --git a/tests/native/first-experiment-steps.ts b/tests/native/first-experiment-steps.ts index 1452db8..ad18643 100644 --- a/tests/native/first-experiment-steps.ts +++ b/tests/native/first-experiment-steps.ts @@ -1,6 +1,7 @@ +import { installOrderSummary } from "./rebuild-install.js"; // The steps of spec:application.first-experiment's example space, shared by its native examples. They // run on the production composition: admin access seeds the grants and runs the order summary's first -// activation as setup, and reads stored documents and the function log; every command and query is +// rebuild as setup, and reads stored documents and the function log; every command and query is // sent by an ordinary client whose token the fixture issuer signs. import type { ConvexHttpClient } from "convex/browser"; import { @@ -29,7 +30,6 @@ import { scheduledRows } from "./scheduled-rows.js"; import { utf8Length, type MutationCtx } from "../../src/command/index.js"; export const tenantId = "t-1"; export const subject = "user-1"; -const operator = { kind: "operator", id: "native-test" } as const; export const grantedPermissions = [ placeOrderPermission, receiveStockPermission, @@ -107,17 +107,14 @@ export async function grant( }); } // The production composition on its own backend, with the setup every run takes before its first -// command: the grants and the order summary's first activation, both with admin access. +// command: the grants and the order summary's first rebuild, both with admin access. export async function application(world: { backend?: Backend; client?: ConvexHttpClient; }) { const backend = await productionBackend(); await grant(backend, subject, grantedPermissions); - await backend.admin.run(getFunctionName(internal.readModels.activate), { - readModel: "orderSummary", - startedBy: operator, - }); + await installOrderSummary(backend); Object.assign(world, { backend, client: ordinaryClient(backend.url, { diff --git a/tests/native/first-experiment.native-acceptance-production-configuration.test.ts b/tests/native/first-experiment.native-acceptance-production-configuration.test.ts index 1499cd6..9c52fdc 100644 --- a/tests/native/first-experiment.native-acceptance-production-configuration.test.ts +++ b/tests/native/first-experiment.native-acceptance-production-configuration.test.ts @@ -43,7 +43,8 @@ const internalFunctions = [ internal.receiving.receiveStockInternal, internal.grants.grant, internal.grants.revoke, - internal.readModels.activate, + internal.rebuild.startGeneration, + internal.rebuild.switchGeneration, ].map((reference) => getFunctionName(reference)); type Answer = { returned: unknown } | { threw: unknown }; interface AcceptanceWorld { diff --git a/tests/native/gate.test.ts b/tests/native/gate.test.ts index 9eee011..e2fc51f 100644 --- a/tests/native/gate.test.ts +++ b/tests/native/gate.test.ts @@ -1,3 +1,4 @@ +import { installFixtureReadModel } from "./rebuild-install.js"; import { getFunctionName, makeFunctionReference, @@ -120,10 +121,7 @@ test("native: a closed tenant refuses a public command, another tenant writes, a test("native: setting restore on the running backend refuses every declared command route without a deploy and clearing it admits the same keys", async () => { const backend = await fixtureBackend(); const client = await clientFor(backend); - await backend.admin.run(getFunctionName(internal.readModels.activate), { - readModel: "documentSummary", - startedBy: { kind: "operator", id: "setup" }, - }); + await installFixtureReadModel(backend, "documentSummary", "setup"); type Write = { name: string; input: Record; diff --git a/tests/native/generation-registry.command-without-generation-fails.test.ts b/tests/native/generation-registry.command-without-generation-fails.test.ts index b04889a..1cd17aa 100644 --- a/tests/native/generation-registry.command-without-generation-fails.test.ts +++ b/tests/native/generation-registry.command-without-generation-fails.test.ts @@ -12,7 +12,7 @@ import type { LogMark } from "../../harness/admin.js"; import { required } from "../../harness/native.js"; import { classifyThrown } from "../../src/command/index.js"; import { - activateOrderSummary, + installOrderSummary, caught, changedDocuments, line, @@ -53,7 +53,7 @@ bindExample(contract, (): World => ({}), { world, { generationRows }, ) => { - const order = await orderWorld({ activate: false }); + const order = await orderWorld({ install: false }); world.order = order; expect(await order.backend.admin.readTable("generations")).toHaveLength( generationRows, @@ -102,7 +102,7 @@ bindExample(contract, (): World => ({}), { "orders.events": [], orderSummaries: [], }); - await activateOrderSummary(order.backend); + await installOrderSummary(order.backend); expect( await order.client.mutation(api.ordering.placeOrder, call), ).toMatchObject({ kind: "applied", replayed: false }); diff --git a/tests/native/migration-state.ts b/tests/native/migration-state.ts index 2039bd0..841f5b1 100644 --- a/tests/native/migration-state.ts +++ b/tests/native/migration-state.ts @@ -13,6 +13,9 @@ export async function migrationState(backend: Backend, name: string) { const generation = (await backend.admin.readTable("generations")).find( (row) => row.generation === 2, )!; + const progress = (await backend.admin.readTable("generationProgress")).find( + (row) => row.generationId === generation._id, + )!; const rows = (await backend.admin.readTable("documentSummaries")).filter( (row) => row.generation === 2, ); @@ -22,6 +25,7 @@ export async function migrationState(backend: Backend, name: string) { component, worker, generation, + progress, rows, visits, }; @@ -29,7 +33,8 @@ export async function migrationState(backend: Backend, name: string) { export function equalContextCursor( state: Awaited>, ) { - expect((state.generation.cursor as { pageCursor: string }).pageCursor).toBe( + // generation-registry.sdp.md:74: the cursor is held by the progress row. + expect((state.progress.cursor as { pageCursor: string }).pageCursor).toBe( state.component.cursor, ); } diff --git a/tests/native/migration-steps.ts b/tests/native/migration-steps.ts index 2523e22..c4f1f5d 100644 --- a/tests/native/migration-steps.ts +++ b/tests/native/migration-steps.ts @@ -1,3 +1,4 @@ +import { installFixtureReadModel } from "./rebuild-install.js"; import { setTimeout as sleep } from "node:timers/promises"; import { expect } from "vitest"; import type { MigrationStatus } from "@convex-dev/migrations"; @@ -17,17 +18,32 @@ export async function setup(rows: number) { "convex-helpers": "0.1.124", "@convex-dev/migrations": "0.3.6", }); - await backend.admin.run("readModels:activate", { - readModel: "documentSummary", - startedBy: { kind: "operator", id: "native-test" }, - }); + await installFixtureReadModel(backend, "documentSummary"); const generation = (await backend.admin.readTable("generations"))[0]!; const { _id, _creationTime, switchedAt, ...fields } = generation; void _id; void _creationTime; void switchedAt; await backend.admin.writeTable("generations", { - insert: { ...fields, generation: 2, state: "building", batchSize: 2 }, + insert: { ...fields, generation: 2, state: "building" }, + }); + const building = (await backend.admin.readTable("generations")).find( + (row) => row.generation === 2, + )!; + // generation-registry.sdp.md:74: the checkpoint belongs to the progress row. + await backend.admin.writeTable("generationProgress", { + insert: { + generationId: building._id!, + pass: "backfill", + batchSize: 2, + cursor: null, + batchesDone: 0, + rowsWritten: 0, + rowsSkipped: 0, + misses: 0, + rowsPurged: 0, + updatedAt: Date.now(), + }, }); await backend.admin.run("grants:grant", { tenantId: "tenant", diff --git a/tests/native/order-use-case.ts b/tests/native/order-use-case.ts index 4675a98..d34184f 100644 --- a/tests/native/order-use-case.ts +++ b/tests/native/order-use-case.ts @@ -1,11 +1,12 @@ // Shared steps of the scenarios that run PlaceOrder and CancelOrder on the production composition: a backend whose -// user holds every grant the use cases need, the order summary's first activation, stock through +// user holds every grant the use cases need, the order summary's first rebuild, stock through // ReceiveStock, and every stored document of both contexts and the parent, to count what a call // changed. Admin access is used for setup and for reading stored documents only. import { getFunctionName } from "convex/server"; import type { ConvexHttpClient } from "convex/browser"; import type { Value } from "convex/values"; import { api, internal } from "../../example/convex/_generated/api.js"; +import { installOrderSummary } from "./rebuild-install.js"; import { cancelOrderPermission, placeOrderPermission, @@ -44,22 +45,17 @@ export function grant( grantedBy: "native-test", }); } -// The order summary's first activation, run by an operator with admin access. -export function activateOrderSummary(backend: Backend) { - return backend.admin.run(getFunctionName(internal.readModels.activate), { - readModel: "orderSummary", - startedBy: { kind: "operator", id: "native-test" }, - }); -} +// The order summary's first rebuild, run by an operator with admin access. +export { installOrderSummary }; // A production backend, the user's grants and, unless asked not to, the order summary's first -// activation, all with admin access, and an ordinary client with the user's fixture-issuer token. +// rebuild, all with admin access, and an ordinary client with the user's fixture-issuer token. export async function orderWorld( - options: { activate?: boolean } = {}, + options: { install?: boolean } = {}, ): Promise { const backend = await productionBackend(); for (const permission of permissions) await grant(backend, subject, permission); - if (options.activate ?? true) await activateOrderSummary(backend); + if (options.install ?? true) await installOrderSummary(backend); const token = await backend.issuer.token(subject); return { backend, token, client: ordinaryClient(backend.url, { token }) }; } diff --git a/tests/native/production-orders.test.ts b/tests/native/production-orders.test.ts index 24012c0..04cdbb8 100644 --- a/tests/native/production-orders.test.ts +++ b/tests/native/production-orders.test.ts @@ -1,3 +1,4 @@ +import { installOrderSummary } from "./rebuild-install.js"; import { getFunctionName, type PaginationResult } from "convex/server"; import { ConvexError, convexToJson, type Value } from "convex/values"; import { expect, onTestFinished, test } from "vitest"; @@ -14,9 +15,8 @@ import { import { measure, productionBackend, required } from "../../harness/native.js"; import { classifyThrown } from "../../src/command/index.js"; // The production composition on its own backend, reached by ordinary clients whose tokens the -// fixture issuer signs: grants and the order summary's first activation with admin access as setup, +// fixture issuer signs: grants and the order summary's first rebuild with admin access as setup, // stock through ReceiveStock, then PlaceOrder over Orders and Inventory. -const operator = { kind: "operator", id: "native-test" } as const; const readInventoryPermission = "inventory.read"; const everything = [ placeOrderPermission, @@ -39,15 +39,11 @@ async function grant( grantedBy: "native-test", }); } -const activate = (backend: Backend) => - backend.admin.run(getFunctionName(internal.readModels.activate), { - readModel: "orderSummary", - startedBy: operator, - }); -// Grants, the first activation and a client for a granted user. +const install = (backend: Backend) => installOrderSummary(backend); +// Grants, the first rebuild and a client for a granted user. async function setUp(backend: Backend) { await grant(backend, "user-1", everything); - await activate(backend); + await install(backend); const token = await backend.issuer.token("user-1"); return { token, client: ordinaryClient(backend.url, { token }) }; } diff --git a/tests/native/read-models.test.ts b/tests/native/read-models.test.ts index 2cd64fc..ff1b37d 100644 --- a/tests/native/read-models.test.ts +++ b/tests/native/read-models.test.ts @@ -1,3 +1,4 @@ +import { installFixtureReadModel } from "./rebuild-install.js"; import { getFunctionName, type PaginationResult } from "convex/server"; import { ConvexError, convexToJson, type Value } from "convex/values"; import type { ConvexHttpClient } from "convex/browser"; @@ -14,15 +15,12 @@ import { import { fixtureBackend, measure, required } from "../../harness/native.js"; import { classifyThrown } from "../../src/command/index.js"; // The fixture composition's read model, documentSummary: written by CreateSummarizedDocument at the -// pipeline's step 9, made writable by the first activation, and read through the parent list +// pipeline's step 9, made writable by the first rebuild, and read through the parent list // listDocumentSummaries, which authorizes before it reads. const identifier = "depotCommands:createSummarizedDocument"; -const operator = { kind: "operator", id: "native-test" } as const; -const activate = (backend: Backend) => - backend.admin.run(getFunctionName(internal.readModels.activate), { - readModel: "documentSummary", - startedBy: operator, - }); +const operator = "native-test"; +const install = (backend: Backend) => + installFixtureReadModel(backend, "documentSummary"); async function grant( backend: Backend, subject: string, @@ -53,7 +51,7 @@ async function stored(backend: Backend) { }; } -test("native: a command whose read model has no generation fails as a technical failure and stores nothing; after the first activation the same command writes its summary row in its one mutation, and a second activation is refused", async () => { +test("native: a command whose read model has no generation fails as a technical failure and stores nothing; after the first rebuild the same command writes its summary row in its one mutation, and another generation can be built", async () => { const backend = await fixtureBackend(); await grant(backend, "user-1", permissions.documents); const client = ordinaryClient(backend.url, { @@ -79,7 +77,8 @@ test("native: a command whose read model has no generation fails as a technical events: [], summaries: [], }); - expect(await activate(backend)).toBe(1); + // generation-registry.sdp.md:37; rebuild.sdp.md:115,133: the first rebuild installs generation 1. + await install(backend); const registry = await backend.admin.readTable("generations"); expect(registry).toMatchObject([ { @@ -90,11 +89,29 @@ test("native: a command whose read model has no generation fails as a technical startedBy: operator, }, ]); - const second = await caught(activate(backend)); - expect(String(second)).toContain( - "Read model documentSummary already has generation 1, which is active", + // rebuild.sdp.md:115: an active generation does not refuse the next rebuild. + const nextId = await backend.admin.run( + getFunctionName(internal.rebuild.startGeneration), + { + readModel: "documentSummary", + operator, + }, ); - expect(await backend.admin.readTable("generations")).toEqual(registry); + const next = (await backend.admin.readTable("generations")).find( + (row) => row._id === nextId, + ); + // rebuild.sdp.md:115: start takes the next generation number. + expect(next).toMatchObject({ + generation: 2, + projectionVersion: 1, + startedBy: operator, + }); + // Stop its chain before measuring the single command below. + await backend.admin.run(getFunctionName(internal.rebuild.abortGeneration), { + generationId: nextId, + operator, + reason: "keep one writable generation", + }); // The same command and key again: the failed attempt left no receipt, so this is new intent. const mark = await backend.admin.logMark(); const response = await client.mutation( @@ -155,7 +172,7 @@ test("native: a command whose read model has no generation fails as a technical test("native: a refused parent query throws the rejection shape to an HTTP client and writes nothing, and the granted caller reads", async () => { const backend = await fixtureBackend(); - await activate(backend); + await install(backend); await grant(backend, "user-1", permissions.documents); const writer = ordinaryClient(backend.url, { token: await backend.issuer.token("user-1"), @@ -229,7 +246,7 @@ test("native: a refused parent query throws the rejection shape to an HTTP clien test("native: a parent list over the read model pages by the cursor pair, each page read once and then subscribed to its end cursor, with no page split and none of another tenant's rows", async () => { const backend = await fixtureBackend(); - await activate(backend); + await install(backend); for (const tenantId of ["t-1", "t-2"]) await grant(backend, "user-1", permissions.documents, tenantId); await grant(backend, "user-1", readPermission); diff --git a/tests/native/rebuild-install.ts b/tests/native/rebuild-install.ts new file mode 100644 index 0000000..cf8c6e1 --- /dev/null +++ b/tests/native/rebuild-install.ts @@ -0,0 +1,114 @@ +import { getFunctionName } from "convex/server"; +import type { Value } from "convex/values"; +import { setTimeout as sleep } from "node:timers/promises"; +import { internal as production } from "../../example/convex/_generated/api.js"; +import { internal as fixture } from "../../fixture/convex/_generated/api.js"; +import type { Backend } from "../../harness/backend.js"; + +export type Generation = Record & { + _id: string; + generation: number; + state: string; + fence: number; +}; +export type Progress = Record & { + batchesDone: number; + batchSize: number; + pass: string; + cursor: { tenantId: string; pageCursor: string | null } | null; +}; +export type GenerationEntry = { generation: Generation; progress: Progress }; + +// rebuild.sdp.md:137: the operator query returns each generation with its progress row. +export async function generations(backend: Backend, readModel: string) { + return (await backend.admin.run("rebuild:getGenerations", { + readModel, + })) as unknown as GenerationEntry[]; +} +export async function generationEntry( + backend: Backend, + readModel: string, + generationId: string, +) { + const entry = (await generations(backend, readModel)).find( + ({ generation }) => generation._id === generationId, + ); + if (!entry) + throw new Error(`Missing generation ${generationId} of ${readModel}`); + return entry; +} + +// rebuild.online-rebuild-interrupt-resume.sdp.md:33: a running chain has a 30 s stall bound. +export async function waitForGeneration( + backend: Backend, + readModel: string, + generationId: string, + until: (entry: GenerationEntry) => boolean = ({ generation }) => + generation.state === "verified", + observe: (entry: GenerationEntry) => void | Promise = () => {}, +): Promise { + const deadline = Date.now() + 240_000; + let advancedAt = Date.now(); + let batches = -1; + for (;;) { + const entry = await generationEntry(backend, readModel, generationId); + await observe(entry); + if (until(entry)) return entry; + if (entry.progress.batchesDone !== batches) { + batches = entry.progress.batchesDone; + advancedAt = Date.now(); + } + if (Date.now() - advancedAt >= 30_000 || Date.now() >= deadline) + throw new Error(`Generation did not advance: ${JSON.stringify(entry)}`); + await sleep(10); + } +} + +// generation-registry.sdp.md:37; rebuild.sdp.md:115,127,133: install by the first rebuild. +async function install( + backend: Backend, + readModel: string, + refs: { start: string; switch: string }, + operator: string, +) { + const id = await backend.admin.run(refs.start, { + readModel, + projectionVersion: 1, + operator, + }); + if (typeof id !== "string") + throw new Error("startGeneration did not return an ID"); + await waitForGeneration(backend, readModel, id); + await backend.admin.run(refs.switch, { generationId: id, operator }); + return id; +} +export function installOrderSummary( + backend: Backend, + operator = "native-test", +) { + return install( + backend, + "orderSummary", + { + start: getFunctionName(production.rebuild.startGeneration), + switch: getFunctionName(production.rebuild.switchGeneration), + }, + operator, + ); +} +export function installFixtureReadModel( + backend: Backend, + readModel = "documentSummary", + operator = "native-test", +) { + // native-harness.sdp.md:90: documentTitle setups explicitly select version 1. + return install( + backend, + readModel, + { + start: getFunctionName(fixture.rebuild.startGeneration), + switch: getFunctionName(fixture.rebuild.switchGeneration), + }, + operator, + ); +} diff --git a/tests/native/scheduler-cases.ts b/tests/native/scheduler-cases.ts index 82bede5..021f722 100644 --- a/tests/native/scheduler-cases.ts +++ b/tests/native/scheduler-cases.ts @@ -1,3 +1,4 @@ +import { installOrderSummary } from "./rebuild-install.js"; import { expect } from "vitest"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; @@ -286,10 +287,7 @@ export async function scanCase(backend: Backend) { async function productionData(backend: Backend) { await grant(backend, subject, grantedPermissions); - await backend.admin.run("readModels:activate", { - readModel: "orderSummary", - startedBy: { kind: "operator", id: "native-test" }, - }); + await installOrderSummary(backend); const client = ordinaryClient(backend.url, { token: await backend.issuer.token(subject), }); diff --git a/tests/pure/read-model.test.ts b/tests/pure/read-model.test.ts index b3b63db..0feb980 100644 --- a/tests/pure/read-model.test.ts +++ b/tests/pure/read-model.test.ts @@ -23,8 +23,7 @@ const anchorProjection = codeAnchor({ }); const anchorRegistry = codeAnchor({ id: codeAnchorId("impl:application.generation-registry"), - label: - "the generations table, activateFirstGeneration, generationsToWrite and activeGeneration", + label: "the generations table, generationsToWrite and activeGeneration", satisfies: ref("spec:application.generation-registry"), }); const anchorReadModels = codeAnchor({ @@ -98,7 +97,7 @@ test("pure: the fixture's projection keys a summary by document ID and keeps its amendments: 2, version, }; - const { projection } = documentSummary; + const projection = documentSummary.projections[0]!; expect(projection.keyOf("t-1", dto)).toBe("doc-1"); expect(projection.project("t-1", dto, [version])).toStrictEqual({ documentId: "doc-1", diff --git a/tests/simulator/example.test.ts b/tests/simulator/example.test.ts index 12d95f0..7d41a2b 100644 --- a/tests/simulator/example.test.ts +++ b/tests/simulator/example.test.ts @@ -1,3 +1,4 @@ +import { installOrderSummary } from "./gate-support.js"; import { ConvexError, type Value } from "convex/values"; import { readFileSync } from "node:fs"; import { join } from "node:path"; @@ -53,11 +54,7 @@ const everything = [ readOrdersPermission, "inventory.read", ]; -const activate = (t: App) => - t.mutation(internal.readModels.activate, { - readModel: "orderSummary", - startedBy: operator, - }); +const install = (t: App) => installOrderSummary(t); async function failure(promise: Promise): Promise { return promise.then( () => { @@ -86,7 +83,7 @@ test( ), async () => { const t = productionTest(); - await activate(t); + await install(t); const user = await caller(t, "user-1", everything); const received = await user.mutation(api.receiving.receiveStock, { tenantId: "t-1", @@ -200,7 +197,7 @@ test( ), async () => { const t = productionTest(); - await activate(t); + await install(t); const user = await caller(t, "user-1", everything); await user.mutation(api.receiving.receiveStock, { tenantId: "t-1", @@ -286,7 +283,11 @@ test( commandType: "PlaceOrder", message: "An order needs at least one line", }); - expect(await activate(t)).toBe(1); + // rebuild.sdp.md:115,133: installing returns a generation ID; its number is on the row. + await install(t); + expect( + await t.run((ctx) => ctx.db.query("generations").collect()), + ).toMatchObject([{ generation: 1, state: "active" }]); expect(await send([line("sku-1", 1)])).toMatchObject({ kind: "applied" }); }, ); @@ -373,16 +374,17 @@ test( message: "A quantity must be a positive whole number, not 0", details: { quantity: 0 }, }); + // rebuild.sdp.md:115: an unknown read model is refused with the exact message. expect( String( await failure( - t.mutation(internal.readModels.activate, { + t.mutation(internal.rebuild.startGeneration, { readModel: "orderTotals", - startedBy: operator, + operator: "operator-1", }), ), ), - ).toContain("This deployment declares no read model orderTotals"); + ).toBe("Error: This deployment declares no read model orderTotals"); }, ); @@ -476,7 +478,7 @@ test( // Stock of 5 on sku-1, and order-1 placed for 3 of them over two lines, by a caller with every grant. async function placedOrder(t: App) { - await activate(t); + await install(t); const user = await caller(t, "user-1", everything); await user.mutation(api.receiving.receiveStock, { tenantId: "t-1", @@ -777,7 +779,7 @@ test( ), async () => { const t = productionTest(); - await activate(t); + await install(t); const user = await placedOverTwoItems(t, "t-1"); expect(await stockItems(t, "t-1")).toMatchObject([ { onHand: 5, allocated: 2, version: { version: 2 } }, @@ -810,7 +812,7 @@ test( ), async () => { const t = productionTest(); - await activate(t); + await install(t); await placedOverTwoItems(t, "t-1"); const user = await placedOverTwoItems(t, "t-2"); const untouched = await stockItems(t, "t-1"); diff --git a/tests/simulator/fixture-faults.test.ts b/tests/simulator/fixture-faults.test.ts index 2b56069..9357087 100644 --- a/tests/simulator/fixture-faults.test.ts +++ b/tests/simulator/fixture-faults.test.ts @@ -1,3 +1,4 @@ +import { installFixtureReadModel } from "./gate-support.js"; import { convexTest } from "convex-test"; import { ConvexError } from "convex/values"; import { readFileSync } from "node:fs"; @@ -140,10 +141,7 @@ test( async () => { const t = app(); for (const readModel of ["documentSummary", "documentTitle"]) - await t.mutation(internal.readModels.activate, { - readModel, - startedBy: { kind: "operator", id: "operator-1" }, - }); + await installFixtureReadModel(t, readModel); const documentVersion = { tenantId: "t-1", contextId: "depot", diff --git a/tests/simulator/gate-pipeline.test.ts b/tests/simulator/gate-pipeline.test.ts index 9834b76..3f5ea6d 100644 --- a/tests/simulator/gate-pipeline.test.ts +++ b/tests/simulator/gate-pipeline.test.ts @@ -1,3 +1,4 @@ +import { installFixtureReadModel } from "./gate-support.js"; import { v } from "convex/values"; import { afterEach, beforeEach, expect, test, vi } from "vitest"; import { @@ -67,10 +68,7 @@ async function stored(t: GateApp) { test("convex-test: a paused audited command stores no receipt, event, state, read-model row or audit and its key applies after resume", async () => { const t = gateApp(); await caller(t); - await t.mutation(internal.readModels.activate, { - readModel: "documentSummary", - startedBy: { kind: "operator", id: "admin" }, - }); + await installFixtureReadModel(t, "documentSummary", "admin"); const declaration: CommandDeclaration< { documentId: string; title: string }, null diff --git a/tests/simulator/gate-support.ts b/tests/simulator/gate-support.ts index c5289a9..d3e9bd0 100644 --- a/tests/simulator/gate-support.ts +++ b/tests/simulator/gate-support.ts @@ -6,7 +6,9 @@ import { type FunctionReference, } from "convex/server"; import { ConvexError, v, type Value } from "convex/values"; -import { expect } from "vitest"; +import { expect, vi } from "vitest"; +import { internal as production } from "../../example/convex/_generated/api.js"; +import type { productionTest } from "./production.js"; import { internal } from "../../fixture/convex/_generated/api.js"; import annexSchema from "../../fixture/convex/annex/schema.js"; import depotSchema from "../../fixture/convex/depot/schema.js"; @@ -90,14 +92,8 @@ export async function refusal( }); } export async function generationIds(t: GateApp) { - await t.mutation(internal.readModels.activate, { - readModel: "documentSummary", - startedBy: { kind: "operator", id: "setup" }, - }); - await t.mutation(internal.readModels.activate, { - readModel: "documentTitle", - startedBy: { kind: "operator", id: "setup" }, - }); + await installFixtureReadModel(t, "documentSummary", "setup"); + await installFixtureReadModel(t, "documentTitle", "setup"); const rows = await t.run((ctx) => ctx.db.query("generations").collect()); const [first, second] = rows; if (!first || !second) throw new Error("Two generation rows are required"); @@ -120,3 +116,51 @@ export const inspection = ( ).depot.gateInspection.stored; export const depotRows = (t: GateApp) => t.run((ctx) => ctx.runQuery(inspection, {})); + +// generation-registry.sdp.md:37; rebuild.sdp.md:115,127,133: scheduled batches finish before switch. +export async function installFixtureReadModel( + t: GateApp, + readModel = "documentSummary", + operator = "operator-1", +) { + const ownedTimers = !vi.isFakeTimers(); + if (ownedTimers) vi.useFakeTimers(); + try { + // native-harness.sdp.md:90: documentTitle setups select version 1. + const generationId = await t.mutation(internal.rebuild.startGeneration, { + readModel, + projectionVersion: 1, + operator, + }); + await t.finishAllScheduledFunctions(() => vi.runOnlyPendingTimers(), 1000); + await t.mutation(internal.rebuild.switchGeneration, { + generationId, + operator, + }); + return generationId; + } finally { + if (ownedTimers) vi.useRealTimers(); + } +} +export async function installOrderSummary( + t: ReturnType, + operator = "operator-1", +) { + const ownedTimers = !vi.isFakeTimers(); + if (ownedTimers) vi.useFakeTimers(); + try { + const generationId = await t.mutation(production.rebuild.startGeneration, { + readModel: "orderSummary", + projectionVersion: 1, + operator, + }); + await t.finishAllScheduledFunctions(() => vi.runOnlyPendingTimers(), 1000); + await t.mutation(production.rebuild.switchGeneration, { + generationId, + operator, + }); + return generationId; + } finally { + if (ownedTimers) vi.useRealTimers(); + } +} diff --git a/tests/simulator/read-model.test.ts b/tests/simulator/read-model.test.ts index f472bc4..53eaa72 100644 --- a/tests/simulator/read-model.test.ts +++ b/tests/simulator/read-model.test.ts @@ -1,8 +1,9 @@ +import { installFixtureReadModel } from "./gate-support.js"; import { convexTest } from "convex-test"; import { ConvexError, getConvexSize, v, type Value } from "convex/values"; import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { describe, expect, test } from "vitest"; +import { describe, expect, test, vi } from "vitest"; import { api, internal } from "../../fixture/convex/_generated/api.js"; import annexSchema from "../../fixture/convex/annex/schema.js"; import depotSchema from "../../fixture/convex/depot/schema.js"; @@ -54,7 +55,7 @@ function app() { } type App = ReturnType; const issuer = "https://fixture-issuer.test"; -const operator = { kind: "operator", id: "operator-1" } as const; +const operator = "operator-1"; async function caller(t: App, subject: string, permission: string) { await t.mutation(internal.grants.grant, { tenantId: "t-1", @@ -65,8 +66,8 @@ async function caller(t: App, subject: string, permission: string) { }); return t.withIdentity({ issuer, subject }); } -const activate = (t: App, readModel = "documentSummary") => - t.mutation(internal.readModels.activate, { readModel, startedBy: operator }); +const install = (t: App, readModel = "documentSummary") => + installFixtureReadModel(t, readModel); async function failure(promise: Promise): Promise { return promise.then( () => { @@ -184,8 +185,7 @@ type GenerationState = | "retired" | "aborted" | "purged"; -// A registry row in the given state, written directly: only the first activation exists as a -// transition, and the reads must be shown against every state. +// generation-registry.sdp.md:67: seed each state to exercise registry reads independently of transitions. const generationRow = (generation: number, state: GenerationState) => ({ readModel: "documentSummary", @@ -193,15 +193,11 @@ const generationRow = (generation: number, state: GenerationState) => projectionVersion: 1, state, pauseRequired: false, - batchSize: 0, fence: 0, - batchesDone: 0, - rowsWritten: 0, - rowsSkipped: 0, - misses: 0, startedAt: 0, startedBy: operator, - updatedAt: 0, + changedAt: 0, + changedBy: operator, }) as const; async function registry(t: App, rows: [number, GenerationState][]) { await t.run(async (ctx) => { @@ -210,49 +206,74 @@ async function registry(t: App, rows: [number, GenerationState][]) { }); } -describe("the first activation", () => { +describe("the first rebuild", () => { test( name( - "inserts generation 1 as active with the projection's version, and a second activation is refused with the generation and its state", + "installs generation 1 and refuses another start while a generation is in flight", ), async () => { const t = app(); - expect(await activate(t)).toBe(1); + const generationId = await install(t); + // generation-registry.sdp.md:37; rebuild.sdp.md:115,133: installation is the first rebuild. expect(await generations(t)).toMatchObject([ { - readModel: "documentSummary", + _id: generationId, generation: 1, projectionVersion: 1, state: "active", - pauseRequired: false, startedBy: operator, - batchesDone: 0, - rowsWritten: 0, }, ]); - await technicalFailure( - activate(t), - "Read model documentSummary already has generation 1, which is active", - ); - expect(await generations(t)).toHaveLength(1); + vi.useFakeTimers(); + try { + const second = await t.mutation(internal.rebuild.startGeneration, { + readModel: "documentSummary", + operator, + }); + // rebuild.sdp.md:115: active does not refuse a start; building does, by this exact message. + await expect( + t.mutation(internal.rebuild.startGeneration, { + readModel: "documentSummary", + operator, + }), + ).rejects.toThrow( + /^Read model documentSummary already has generation 2 in building$/, + ); + // rebuild.sdp.md:115: the refused start inserted no third generation. + expect(await generations(t)).toHaveLength(2); + await t.finishAllScheduledFunctions(() => vi.runOnlyPendingTimers()); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: second, + operator, + }); + } finally { + vi.useRealTimers(); + } }, ); - test( name( - "is refused for a read model with a row in any other state, and for a name the composition does not declare", + "starts after an aborted generation and refuses an undeclared read model", ), async () => { const t = app(); await registry(t, [[3, "aborted"]]); - await technicalFailure( - activate(t), - "Read model documentSummary already has generation 3, which is aborted", - ); - await technicalFailure( - activate(t, "nope"), - "This deployment declares no read model nope", + const generationId = await install(t); + // rebuild.sdp.md:115: the next generation is one above the last, including an aborted one. + expect(await generations(t)).toContainEqual( + expect.objectContaining({ + _id: generationId, + generation: 4, + state: "active", + }), ); + // rebuild.sdp.md:115: an unknown read model is refused before registration. + await expect( + t.mutation(internal.rebuild.startGeneration, { + readModel: "nope", + operator, + }), + ).rejects.toThrow(/^This deployment declares no read model nope$/); }, ); }); @@ -287,8 +308,12 @@ describe("the registry's reads", () => { [4, state], [3, "active"], ]); + // generation-registry.sdp.md:81: verifying and verified use the verifying role. expect(await read(other)).toEqual({ - write: [writable("active", 3), writable("building", 4)], + write: [ + writable("active", 3), + writable(state === "building" ? "building" : "verifying", 4), + ], query: 3, }); } @@ -331,7 +356,7 @@ describe("step 9", () => { ), async () => { const t = app(); - await activate(t); + await install(t); const user = await caller(t, "user-1", permissions.documents); const response = await user.mutation( api.depotCommands.createSummarizedDocument, @@ -397,7 +422,7 @@ describe("step 9", () => { ), async () => { const t = app(); - await activate(t); + await install(t); const stock = documentEntry("p-1", 1, "x", "stock"); for (const readModels of [ [], @@ -426,7 +451,7 @@ describe("step 9", () => { ), async () => { const t = app(); - await activate(t); + await install(t); const entries = (count: number) => Array.from({ length: count }, (_, i) => documentEntry(`doc-${i}`)); const binding = { readModel: documentSummary, source: documentSource }; @@ -438,17 +463,23 @@ describe("step 9", () => { expect(await summaries(t)).toEqual([]); // A projection that answers no row for one of five entries writes four rows. const skipping = variant({ - projection: { - ...documentSummary.projection, - project: (tenantId, dto, versions) => - dto["documentId"] === "doc-0" - ? null - : documentSummary.projection.project( - tenantId, - dto as never, - versions, - ), - }, + projections: [ + { + ...documentSummary.projections[0]!, + project: ( + tenantId: string, + dto: Record, + versions: StreamVersion[], + ) => + dto["documentId"] === "doc-0" + ? null + : documentSummary.projections[0]!.project( + tenantId, + dto as never, + versions, + ), + }, + ], }); await run( t, @@ -471,7 +502,7 @@ describe("step 9", () => { ), async () => { const t = app(); - await activate(t); + await install(t); const entry = documentEntry("doc-1", 1, "x".repeat(300)); const bytes = getConvexSize({ documentId: "doc-1", @@ -592,7 +623,9 @@ describe("applyProjection", () => { async () => { const t = app(); const none = variant({ - projection: { ...documentSummary.projection, project: () => null }, + projections: [ + { ...documentSummary.projections[0]!, project: () => null }, + ], }); await apply(t, documentEntry("doc-1").dto, "live-created", [ active as WritableGeneration, @@ -611,17 +644,19 @@ describe("applyProjection", () => { ); test( - name( - "a generation is written with the declared projection whatever projection version it records, and the row records the projection's", - ), + name("an undeclared projection version is refused before a row is written"), async () => { const t = app(); - await apply(t, documentEntry("doc-1").dto, "live-created", [ - { role: "active", generation: 1, projectionVersion: 7 }, - ]); - expect(await summaries(t)).toMatchObject([ - { generation: 1, projectionVersion: 1 }, - ]); + // projection-contract.sdp.md:68: projectionOf refuses a version absent from projections. + await expect( + apply(t, documentEntry("doc-1").dto, "live-created", [ + { role: "active", generation: 1, projectionVersion: 7 }, + ]), + ).rejects.toThrow( + /^Read model documentSummary declares no projection of version 7$/, + ); + // projection-contract.sdp.md:68: an undeclared projection writes no row. + expect(await summaries(t)).toEqual([]); }, ); }); From bac760be9867a9544614c55794c6e496a305eb8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:12:38 +0200 Subject: [PATCH 082/155] build: 3b the first rebuild installs a read model, the example renamed from the first activation Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- design/specs/application.pack.sdp.md | 2 +- ....first-rebuild-installs-read-model.sdp.md} | 10 +- ...tivation-makes-read-model-writable.test.ts | 38 --- ....first-rebuild-installs-read-model.test.ts | 226 ++++++++++++++++++ 5 files changed, 233 insertions(+), 45 deletions(-) rename design/specs/application/{generation-registry.first-activation-makes-read-model-writable.sdp.md => generation-registry.first-rebuild-installs-read-model.sdp.md} (54%) delete mode 100644 tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts create mode 100644 tests/native/generation-registry.first-rebuild-installs-read-model.test.ts diff --git a/design/README.md b/design/README.md index fef6bf4..20325fd 100644 --- a/design/README.md +++ b/design/README.md @@ -317,7 +317,7 @@ The Layer 2 design added two more for the same reason, and ten examples that ver |---|---|---| | L2-2 | `spec:application.read-models.committed-state-read-through-query` | the row says "a query or subscription" | | L2-4 | `spec:command.command-declaration.broken-argument-contract-fails-build` | the row says "rename a typed handler or break its argument contract" | -| none | `spec:application.generation-registry.first-activation-makes-read-model-writable` | verifies that a read model is installed through its first rebuild, under E-8 | +| none | `spec:application.generation-registry.first-rebuild-installs-read-model` | verifies that a read model is installed through its first rebuild, under E-8 | | none | `spec:application.generation-registry.command-without-generation-fails` | verifies that a command whose read model has no generation fails, under E-8 | | none | `spec:application.read-models.query-refused-before-disclosure` | verifies Law 5 for a parent query called with no identity | | none | `spec:application.read-models.query-refused-without-grant` | verifies Law 5 for a parent query called with no grant in the tenant | diff --git a/design/specs/application.pack.sdp.md b/design/specs/application.pack.sdp.md index 4347cbd..0cdf186 100644 --- a/design/specs/application.pack.sdp.md +++ b/design/specs/application.pack.sdp.md @@ -16,7 +16,7 @@ specs: - spec:application.rebuild.write-pause-rebuild-interrupt - spec:application.rebuild.write-pause-rebuild-abort - spec:application.generation-registry - - spec:application.generation-registry.first-activation-makes-read-model-writable + - spec:application.generation-registry.first-rebuild-installs-read-model - spec:application.generation-registry.command-without-generation-fails - spec:application.write-pause - spec:application.write-pause.restore-door-without-deploy diff --git a/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md b/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md similarity index 54% rename from design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md rename to design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md index 3a67e5e..debbdb6 100644 --- a/design/specs/application/generation-registry.first-activation-makes-read-model-writable.sdp.md +++ b/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md @@ -1,5 +1,5 @@ --- -id: spec:application.generation-registry.first-activation-makes-read-model-writable +id: spec:application.generation-registry.first-rebuild-installs-read-model kind: example altitude: story readiness: defined @@ -9,7 +9,7 @@ relations: --- # A read model is installed through its first rebuild -E-8 · native tier · production composition · no acceptance row; the example verifies that a read model's first generation is built like any other and serves the subjects that already exist, an extension under E-8. +E-8 · native tier · fixture composition · no acceptance row; the example verifies that a read model's first generation is built like any other and serves the subjects that already exist, an extension under E-8. ## Intent @@ -25,8 +25,8 @@ And a second start while generation 1 is being built is {second: "refused"} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. -- The test places three orders through a command that declares no read model, or writes them before the read model is declared, then runs `internal.rebuild.startGeneration` for the order summary with admin access and a stated operator, and asserts one row in `generations`, generation 1, `building`, with the order summary's latest projection version and the operator as `startedBy`. +- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- The test creates three documents through `CreateDocument`, which declares no read model, then runs `internal.rebuild.startGeneration` for `documentSummary` with admin access and a stated operator, and asserts one row in `generations`, generation 1, `building`, with `documentSummary`'s latest projection version and the operator as `startedBy`. - The test runs `startGeneration` again before the first generation is switched and asserts that its error names the read model, generation 1 and its state, and that the table still holds one row. -- The test waits until the row is `verified`, asserts that a query over the read model still fails for want of an active generation, runs `switchGeneration`, and asserts that the row is `active`, that no row is `retired`, and that the list query returns the three orders. +- The test waits until the row is `verified`, asserts that `listDocumentSummaries` still fails for want of an active generation, runs `switchGeneration`, and asserts that the row is `active`, that no row is `retired`, and that `listDocumentSummaries` returns the three documents under their status. - The test runs the same sequence on a backend with one tenant and no subject and asserts that the progress row's `batchesDone` is 2, one backfill batch and one verify batch, before `verified`. diff --git a/tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts b/tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts deleted file mode 100644 index c64c091..0000000 --- a/tests/native/generation-registry.first-activation-makes-read-model-writable.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { - ref, - specTest, - testAnchorId, -} from "@libar-dev/software-delivery-protocol"; -import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; -import { describe } from "vitest"; -import { firstActivationMakesReadModelWritableContract as contract } from "../../generated/contracts/application.generation-registry.first-activation-makes-read-model-writable.contract.js"; -const anchor = specTest({ - id: testAnchorId( - "test:application.generation-registry.first-activation-makes-read-model-writable", - ), - verifies: ref( - "spec:application.generation-registry.first-activation-makes-read-model-writable", - ), -}); -void anchor; -// A read model is installed through its first rebuild: startGeneration, its batches and -// switchGeneration. The rebuild is not built, so the example is bound to its steps and skipped, and -// each step throws until the rebuild's build fills it in. -const notBuilt = (step: string): never => { - throw new Error(`Not built: ${step}`); -}; -type World = Record; -describe.skip("the first rebuild installs a read model", () => { - bindExample(contract, (): World => ({}), { - "a read model with {generationRows} generation rows and {orders} subjects that already have history": - () => notBuilt("a read model with history and no generation row"), - "an operator starts a generation, its batches finish and the operator switches it": - () => notBuilt("startGeneration, its batches and switchGeneration"), - "generation {generation} of the read model is {state}": () => - notBuilt("the generation row after the switch"), - "the read model holds {rows} rows for the subjects that existed before it": - () => notBuilt("the rows of the subjects that existed before"), - "a second start while generation 1 is being built is {second}": () => - notBuilt("the refusal of a second start"), - }); -}); diff --git a/tests/native/generation-registry.first-rebuild-installs-read-model.test.ts b/tests/native/generation-registry.first-rebuild-installs-read-model.test.ts new file mode 100644 index 0000000..c5b4b10 --- /dev/null +++ b/tests/native/generation-registry.first-rebuild-installs-read-model.test.ts @@ -0,0 +1,226 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { getFunctionName } from "convex/server"; +import type { ConvexHttpClient } from "convex/browser"; +import { expect, test } from "vitest"; +import { api, internal } from "../../fixture/convex/_generated/api.js"; +import { readPermission } from "../../fixture/convex/readModels.js"; +import { firstRebuildInstallsReadModelContract as contract } from "../../generated/contracts/application.generation-registry.first-rebuild-installs-read-model.contract.js"; +import type { Backend } from "../../harness/backend.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend, required } from "../../harness/native.js"; +import { giveGrant } from "./tenancy-steps.js"; +import { + generationEntry, + generations, + waitForGeneration, +} from "./rebuild-install.js"; + +const anchor = specTest({ + id: testAnchorId( + "test:application.generation-registry.first-rebuild-installs-read-model", + ), + verifies: ref( + "spec:application.generation-registry.first-rebuild-installs-read-model", + ), +}); +void anchor; +const operator = "document installer"; +const tenantId = "documents"; +interface World { + backend?: Backend; + client?: ConvexHttpClient; + generationId?: string; + second?: string; + documentIds: string[]; +} +async function prepare(world: World, count: number) { + const backend = await fixtureBackend(); + await giveGrant(backend, tenantId, "writer"); + await backend.admin.run(getFunctionName(internal.grants.grant), { + tenantId, + principalKind: "human", + principalId: `${backend.issuer.issuer}|writer`, + permission: readPermission, + grantedBy: operator, + }); + const client = ordinaryClient(backend.url, { + token: await backend.issuer.token("writer"), + }); + Object.assign(world, { backend, client }); + for (let i = 0; i < count; i++) { + const documentId = `document-${i}`; + const result = await client.mutation(api.depotCommands.createDocument, { + tenantId, + input: { documentId, title: documentId }, + }); + // generation-registry.first-rebuild-installs-read-model.sdp.md:29: CreateDocument needs no read model. + expect(result.kind).toBe("applied"); + world.documentIds.push(documentId); + } +} +function list(world: World) { + return required(world.client, "the caller").query( + api.readModels.listDocumentSummaries, + { + tenantId, + status: "draft", + paginationOpts: { cursor: null, numItems: 10 }, + }, + ); +} +export async function plainNativeMessage(call: Promise) { + try { + await call; + } catch (error) { + return ( + String(error).split("Uncaught Error: ")[1]?.split("\n")[0] ?? + String(error) + ); + } + throw new Error("Expected a plain error"); +} +async function rebuild(world: World) { + const backend = required(world.backend, "the backend"); + // rebuild.sdp.md:123: park the batch so the building row can be observed without a scheduler race. + await backend.admin.run(getFunctionName(internal.gate.closeGate), { + scopeKey: `tenant:${tenantId}`, + reason: "observe the first generation", + operator, + }); + const generationId = await backend.admin.run( + getFunctionName(internal.rebuild.startGeneration), + { + readModel: "documentSummary", + operator, + }, + ); + if (typeof generationId !== "string") + throw new Error("Expected a generation ID"); + world.generationId = generationId; + const entries = await generations(backend, "documentSummary"); + // generation-registry.first-rebuild-installs-read-model.sdp.md:29: one building generation with the latest version and stated operator. + expect(entries.map(({ generation }) => generation)).toMatchObject([ + { + _id: generationId, + generation: 1, + state: "building", + projectionVersion: 1, + startedBy: operator, + }, + ]); + // generation-registry.first-rebuild-installs-read-model.sdp.md:29: exactly one generation row. + expect(entries).toHaveLength(1); + world.second = await plainNativeMessage( + backend.admin.run(getFunctionName(internal.rebuild.startGeneration), { + readModel: "documentSummary", + operator, + }), + ); + // generation-registry.first-rebuild-installs-read-model.sdp.md:30; rebuild.sdp.md:115: exact in-flight refusal. + expect(world.second).toBe( + "Read model documentSummary already has generation 1 in building", + ); + // generation-registry.first-rebuild-installs-read-model.sdp.md:30: the refused start adds no row. + expect(await backend.admin.readTable("generations")).toEqual( + entries.map(({ generation }) => generation), + ); + // Let the scheduled batch park before reopening the gate. + await waitForGeneration( + backend, + "documentSummary", + generationId, + ({ progress }) => + progress.lastError === + `write paused for tenant:${tenantId}: observe the first generation`, + ); + await backend.admin.run(getFunctionName(internal.gate.resumeGate), { + scopeKey: `tenant:${tenantId}`, + operator, + }); + await backend.admin.run(getFunctionName(internal.rebuild.resumeGeneration), { + generationId, + operator, + }); + const verified = await waitForGeneration( + backend, + "documentSummary", + generationId, + ); + // generation-registry.first-rebuild-installs-read-model.sdp.md:31: verified still has no active generation. + expect(await plainNativeMessage(list(world))).toBe( + "The read model documentSummary has no active generation", + ); + if (world.documentIds.length === 0) { + // generation-registry.first-rebuild-installs-read-model.sdp.md:32: one empty tenant takes one backfill and one verify batch. + expect(verified.progress).toMatchObject({ pass: "idle", batchesDone: 2 }); + } + await backend.admin.run(getFunctionName(internal.rebuild.switchGeneration), { + generationId, + operator, + }); + // generation-registry.first-rebuild-installs-read-model.sdp.md:31: the first switch retires no generation. + expect( + (await generations(backend, "documentSummary")).filter( + ({ generation }) => generation.state === "retired", + ), + ).toEqual([]); +} +bindExample(contract, (): World => ({ documentIds: [] }), { + "a read model with {generationRows} generation rows and {orders} subjects that already have history": + async (world, { generationRows, orders }) => { + await prepare(world, orders); + // generation-registry.first-rebuild-installs-read-model.sdp.md:29: history exists before the first generation. + expect( + await required(world.backend, "the backend").admin.readTable( + "generations", + ), + ).toHaveLength(generationRows); + }, + "an operator starts a generation, its batches finish and the operator switches it": + rebuild, + "generation {generation} of the read model is {state}": async ( + world, + { generation, state }, + ) => { + const entry = await generationEntry( + required(world.backend, "the backend"), + "documentSummary", + required(world.generationId, "the generation"), + ); + // generation-registry.first-rebuild-installs-read-model.sdp.md:31: first switch makes generation 1 active. + expect(entry.generation).toMatchObject({ generation, state }); + }, + "the read model holds {rows} rows for the subjects that existed before it": + async (world, { rows }) => { + const page = await list(world); + // generation-registry.first-rebuild-installs-read-model.sdp.md:31: all three prior documents appear under draft status. + expect( + page.page.map(({ documentId, status }) => ({ documentId, status })), + ).toEqual( + world.documentIds.map((documentId) => ({ + documentId, + status: "draft", + })), + ); + // generation-registry.first-rebuild-installs-read-model.sdp.md:31: the list returns three documents. + expect(page.page).toHaveLength(rows); + }, + "a second start while generation 1 is being built is {second}": (world) => { + // generation-registry.first-rebuild-installs-read-model.sdp.md:30: the second start is refused during building. + expect(world.second).toBe( + "Read model documentSummary already has generation 1 in building", + ); + }, +}); +test("native: the first rebuild installs an empty read model in two batches", async () => { + const world: World = { documentIds: [] }; + await prepare(world, 0); + await rebuild(world); + // generation-registry.first-rebuild-installs-read-model.sdp.md:32: the same sequence serves the empty tenant. + expect((await list(world)).page).toEqual([]); +}); From b88cc48074267de64705231ee778c63a7a3366f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:12:38 +0200 Subject: [PATCH 083/155] build: 3b the online rebuild interrupted and resumed while commands run Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- ...ld.online-rebuild-interrupt-resume.test.ts | 876 ++++++++++++++++++ 2 files changed, 877 insertions(+), 1 deletion(-) create mode 100644 tests/native/rebuild.online-rebuild-interrupt-resume.test.ts diff --git a/design/README.md b/design/README.md index 20325fd..e4c4127 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -232 specs · 5 packs · 130 anchors → 367 nodes · 1222 edges (0 errors, 0 warnings) +232 specs · 5 packs · 131 anchors → 368 nodes · 1223 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts new file mode 100644 index 0000000..80e10f9 --- /dev/null +++ b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts @@ -0,0 +1,876 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import type { ConvexHttpClient } from "convex/browser"; +import { getFunctionName, type PaginationResult } from "convex/server"; +import type { Value } from "convex/values"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { expect, onTestFinished, vi } from "vitest"; +import { api, internal } from "../../example/convex/_generated/api.js"; +import type { OrderDto } from "../../example/convex/orders/streams.js"; +import { onlineRebuildInterruptResumeContract as contract } from "../../generated/contracts/application.rebuild.online-rebuild-interrupt-resume.contract.js"; +import type { CompletionRecord, LogMark } from "../../harness/admin.js"; +import type { Backend } from "../../harness/backend.js"; +import { + ordinaryClient, + ordinarySocketClient, + watchQuery, +} from "../../harness/clients.js"; +import { productionBackend, required } from "../../harness/native.js"; +import { grant, permissions } from "./order-use-case.js"; +import { + generationEntry, + generations, + installOrderSummary, + waitForGeneration, + type GenerationEntry, +} from "./rebuild-install.js"; + +vi.setConfig({ testTimeout: 300_000 }); +const anchor = specTest({ + id: testAnchorId("test:application.rebuild.online-rebuild-interrupt-resume"), + verifies: ref("spec:application.rebuild.online-rebuild-interrupt-resume"), +}); +void anchor; +const tenants = ["tenant-a", "tenant-z"] as const; +const operator = "order summary operator"; +const readModel = "orderSummary"; +const listName = getFunctionName(api.readModels.listOrderSummaries); +type Row = Record; +type Execution = CompletionRecord & { + willRetry?: boolean; + occInfo?: { + tableName: string; + componentPath: string | null; + retryCount: number; + } | null; +}; +type Operation = { + operationId: string; + orders: number; + inventory: number; + receipts: number; +}; +type Watch = ReturnType< + typeof watchQuery +>; +interface World { + backend?: Backend; + clients: ConvexHttpClient[]; + generation1?: string; + generation2?: string; + initial?: GenerationEntry; + interrupted?: GenerationEntry; + resumed?: GenerationEntry; + verified?: GenerationEntry; + oldFence?: number; + missingOrder?: string | undefined; + existingOrder?: string | undefined; + operations: Operation[]; + log: Map; + mark?: LogMark; + batchStart?: number; + batchEnd?: number; + cursors: string[]; + live: boolean; + workers: Promise[]; + workerError?: unknown; + liveCounts: number[]; + watches: Watch[]; + startedAt: number; + recordPath?: string; +} +const backendOf = (w: World) => required(w.backend, "the backend"); +const id2 = (w: World) => required(w.generation2, "generation 2"); +const seeded = (i: number) => `seed-${String(i).padStart(3, "0")}`; +const records = (w: World) => [...w.log.values()]; + +// Marks overlap: establish the next mark before draining the previous one, then deduplicate by execution. +async function drain(w: World) { + const backend = backendOf(w); + const next = await backend.admin.logMark(); + const previous = w.mark; + if (previous) { + let read = false; + const entries = await backend.admin.completionsSince(previous, () => { + if (!read) { + read = true; + return false; + } + return true; + }); + for (const entry of entries) + w.log.set(entry.executionId, entry as Execution); + } + w.mark = next; +} +async function observe(w: World, entry: GenerationEntry) { + if (entry.progress.cursor && entry.progress.pass === "backfill") + w.cursors.push(entry.progress.cursor.tenantId); + if (w.workerError) throw w.workerError; + await drain(w); +} +async function entry(w: World, generationId = id2(w)) { + return generationEntry(backendOf(w), readModel, generationId); +} +async function place(w: World, tenant: number, orderId: string) { + const response = await w.clients[tenant]!.mutation(api.ordering.placeOrder, { + tenantId: tenants[tenant]!, + requestKey: `place:${orderId}`, + input: { + orderId, + lines: [{ stockItemId: "stock", quantity: 1, unitPrice: 100 }], + }, + }); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: no command ends in a failure. + expect(response).toMatchObject({ kind: "applied", replayed: false }); + w.operations.push({ + operationId: response.operationId, + orders: 1, + inventory: 1, + receipts: 1, + }); +} +async function cancel(w: World, tenant: number, orderId: string) { + const response = await w.clients[tenant]!.mutation(api.ordering.cancelOrder, { + tenantId: tenants[tenant]!, + requestKey: `cancel:${orderId}`, + input: { orderId }, + }); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: no command ends in a failure. + expect(response).toMatchObject({ kind: "applied", replayed: false }); + w.operations.push({ + operationId: response.operationId, + orders: 1, + inventory: 1, + receipts: 1, + }); +} +function startLive(w: World) { + w.live = true; + w.workers = tenants.map(async (_tenant, i) => { + try { + while (w.live) { + const n = w.liveCounts[i]!; + const orderId = `live-${n}`; + await place(w, i, orderId); + await cancel(w, i, orderId); + w.liveCounts[i] = n + 1; + await sleep(200); + } + } catch (error) { + w.workerError = error; + w.live = false; + } + }); +} +async function stopLive(w: World) { + w.live = false; + await Promise.all(w.workers); + if (w.workerError) throw w.workerError; +} +async function rows(w: World, generation: number) { + return (await backendOf(w).admin.readTable("orderSummaries")).filter( + (row) => row.generation === generation, + ); +} +function without(row: Row, keys: string[]) { + return Object.fromEntries( + Object.entries(row).filter(([key]) => !keys.includes(key)), + ); +} +async function verifyBatchCount(w: World, batchSize: number) { + let count = 0; + for (const tenantId of tenants) { + let cursor: string | null = null; + for (;;) { + const page = (await backendOf(w).admin.run( + "queries/order:list", + { + tenantId, + includeDeleted: true, + paginationOpts: { cursor, numItems: batchSize }, + }, + { component: "orders" }, + )) as unknown as PaginationResult; + count++; + await drain(w); + if (page.isDone) break; + cursor = page.continueCursor; + } + } + return count; +} +async function assertCoverage(w: World, generation: number) { + const target = await rows(w, generation); + const expected: { tenantId: string; key: string }[] = []; + for (const tenantId of tenants) { + let cursor: string | null = null; + for (;;) { + const page = (await backendOf(w).admin.run( + "queries/order:list", + { + tenantId, + includeDeleted: true, + paginationOpts: { cursor, numItems: 16 }, + }, + { component: "orders" }, + )) as unknown as PaginationResult; + for (const order of page.page) { + expected.push({ tenantId, key: order.orderId }); + const matching = target.filter( + (row) => row.tenantId === tenantId && row.key === order.orderId, + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: each enumerated order has exactly one row at its source version. + expect(matching).toHaveLength(1); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: no stale data, including a cancel while interrupted. + expect(matching[0]).toMatchObject({ + sourceVersions: [order.version], + projectionVersion: 1, + orderId: order.orderId, + status: order.status, + total: order.total, + lineCount: order.lines.length, + placedAt: order.placedAt, + }); + } + await drain(w); + if (page.isDone) break; + cursor = page.continueCursor; + } + } + const key = ({ + tenantId, + key, + }: { + tenantId: Value | undefined; + key: Value | undefined; + }) => `${tenantId}/${key}`; + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: coverage equals the Orders enumeration in both tenants. + expect( + target.map((row) => key({ tenantId: row.tenantId, key: row.key })).sort(), + ).toEqual(expected.map(key).sort()); +} +async function assertJournals(w: World) { + const backend = backendOf(w); + const tables = { + orders: await backend.admin.readTable("events", { component: "orders" }), + inventory: await backend.admin.readTable("events", { + component: "inventory", + }), + receipts: await backend.admin.readTable("receipts"), + }; + for (const kind of ["orders", "inventory", "receipts"] as const) { + const byOperation = new Map(); + for (const row of tables[kind]) { + const operationId = String(row.operationId); + byOperation.set(operationId, (byOperation.get(operationId) ?? 0) + 1); + } + const expected = new Map( + w.operations + .filter((operation) => operation[kind] > 0) + .map((operation) => [operation.operationId, operation[kind]]), + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: journals and receipts, grouped by operation, contain exactly what the commands produced. + expect(byOperation).toEqual(expected); + } +} +async function assertLog(w: World) { + await drain(w); + const log = records(w); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: no action ran, including outside the observed batch interval. + expect( + log.filter( + ({ udfType }) => udfType === "Action" || udfType === "HttpAction", + ), + ).toEqual([]); + // The harness's own reads through system functions are the test's, not the run's. + const during = log.filter( + (record) => + !record.identifier.startsWith("_system/") && + record.timestamp * 1000 > required(w.batchStart, "batch start") && + record.timestamp * 1000 <= required(w.batchEnd, "batch end"), + ); + const allowed = new Set([ + "rebuild:startGeneration", + "rebuild:interruptGeneration", + "rebuild:resumeGeneration", + "rebuild:backfillBatch", + "rebuild:verifyBatch", + "rebuild:getGenerations", + getFunctionName(api.ordering.placeOrder), + getFunctionName(api.ordering.cancelOrder), + ]); + const mutationNames = new Set([ + ...allowed, + "rebuild:switchGeneration", + "rebuild:rollbackGeneration", + getFunctionName(api.receiving.receiveStock), + ]); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: installation and rollback also have no unaccounted mutation. + expect( + log.filter( + (record) => + record.udfType === "Mutation" && !mutationNames.has(record.identifier), + ), + ).toEqual([]); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: only batches, operator entries and the live commands complete during the batches. + expect(during.filter((record) => !allowed.has(record.identifier))).toEqual( + [], + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: the batch log assertion is not vacuous. + expect( + during.filter( + (record) => + record.identifier === "rebuild:backfillBatch" && !record.willRetry, + ).length, + ).toBeGreaterThanOrEqual(350); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: no completion names progress as its conflict table. + expect( + log.filter((record) => record.occInfo?.tableName === "generationProgress"), + ).toEqual([]); + const commands = new Set([ + getFunctionName(api.ordering.placeOrder), + getFunctionName(api.ordering.cancelOrder), + getFunctionName(api.receiving.receiveStock), + ]); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: every final command completion is successful. + expect( + log.filter( + (record) => + commands.has(record.identifier) && + !record.willRetry && + (record.error !== null || record.occInfo != null), + ), + ).toEqual([]); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: every completed command belongs to a command the test sent, including retries with no new effects. + expect( + log.filter( + (record) => + commands.has(record.identifier) && + !record.willRetry && + record.error === null, + ), + ).toHaveLength(w.operations.length); + const transitions = new Set([ + "rebuild:startGeneration", + "rebuild:interruptGeneration", + "rebuild:resumeGeneration", + "rebuild:switchGeneration", + "rebuild:rollbackGeneration", + ]); + const operatorEntries = log.filter( + (record) => + transitions.has(record.identifier) && + !record.willRetry && + record.error === null, + ).length; + // Two pass ends per installation or rebuild, and one for the rollback verify pass. + const passEnds = w.watches.length === 0 ? 4 : 5; + const reruns = log.filter( + (record) => record.occInfo?.tableName === "generations", + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: two ordinary callers and one chain bound registry reruns per transition. + expect(reruns.length).toBeLessThanOrEqual( + (operatorEntries + passEnds) * (2 + 1), + ); +} +async function subscribe(w: World) { + for (const tenantId of tenants) { + const socket = ordinarySocketClient(backendOf(w).url, { + token: await backendOf(w).issuer.token("writer"), + }); + onTestFinished(() => socket.close()); + const watch = watchQuery(socket, api.readModels.listOrderSummaries, { + tenantId, + status: "placed", + paginationOpts: { cursor: null, numItems: 100 }, + }); + await watch.until(() => true, `the initial list for ${tenantId}`); + w.watches.push(watch); + } + await drain(w); +} +async function switchAndObserve(w: World, generationId: string) { + const backend = backendOf(w); + const before = w.watches.map((watch) => watch.values.at(-1)!.page); + // No commands or explicit list queries occur in this interval. The two query arguments name different tenants. + const mark = await backend.admin.logMark(); + await backend.admin.run(getFunctionName(internal.rebuild.switchGeneration), { + generationId, + operator, + }); + const completions = await backend.admin.completionsSince( + mark, + (seen) => + seen.filter( + (record) => + record.identifier === listName && + record.udfType === "Query" && + !record.cachedResult, + ).length >= 2, + ); + for (const completion of completions) + w.log.set(completion.executionId, completion as Execution); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: both subscriptions reexecute because of this switch. + expect( + completions.filter( + (record) => + record.identifier === listName && + record.udfType === "Query" && + !record.cachedResult, + ), + ).toHaveLength(2); + for (let i = 0; i < w.watches.length; i++) { + const value = await w.watches[i]!.until( + (page) => JSON.stringify(page.page) === JSON.stringify(before[i]), + "the same rows after switch", + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: each subscription retains its rows across the switch. + expect(value.page).toEqual(before[i]); + } + await drain(w); +} + +bindExample( + contract, + (): World => ({ + clients: [], + operations: [], + log: new Map(), + cursors: [], + live: false, + workers: [], + liveCounts: [0, 0], + watches: [], + startedAt: Date.now(), + }), + { + "a {viewKind} read model with an active generation": async (w) => { + const directory = await mkdtemp(join(tmpdir(), "online-rebuild-")); + w.recordPath = join(directory, "run.json"); + console.log(`Online rebuild run record: ${w.recordPath}`); + onTestFinished(async () => { + w.live = false; + await Promise.all(w.workers); + const elapsedMs = Date.now() - w.startedAt; + await writeFile( + w.recordPath!, + JSON.stringify( + { + elapsedMs, + exceeds60Seconds: elapsedMs > 60_000, + callersInFlight: 2, + cursors: w.cursors, + operations: w.operations, + completions: records(w), + registryReruns: records(w).filter( + (record) => record.occInfo?.tableName === "generations", + ), + }, + null, + 2, + ), + ); + console.log( + `Online rebuild elapsed: ${elapsedMs} ms; run record: ${w.recordPath}`, + ); + }); + w.backend = await productionBackend(); + // Deliberately grant in reverse order: the batch must use tenant-ID order. + for (const tenantId of [...tenants].reverse()) + for (const permission of permissions) + await grant(backendOf(w), "writer", permission, tenantId); + w.mark = await backendOf(w).admin.logMark(); + w.generation1 = await installOrderSummary(backendOf(w), operator); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: generation 1 is installed before any order. + expect( + (await generationEntry(backendOf(w), readModel, w.generation1)) + .generation, + ).toMatchObject({ generation: 1, state: "active" }); + await drain(w); + for (let i = 0; i < tenants.length; i++) { + const client = ordinaryClient(backendOf(w).url, { + token: await backendOf(w).issuer.token("writer"), + }); + w.clients.push(client); + const response = await client.mutation(api.receiving.receiveStock, { + tenantId: tenants[i]!, + input: { items: [{ stockItemId: "stock", quantity: 10000 }] }, + }); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: stock setup also ends applied. + expect(response.kind).toBe("applied"); + w.operations.push({ + operationId: response.operationId, + orders: 0, + inventory: 1, + receipts: 0, + }); + for (let n = 0; n < (i === 0 ? 250 : 100); n++) { + await place(w, i, seeded(n)); + if (n % 20 === 0) await drain(w); + } + } + await drain(w); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: exactly 250 and 100 seeded orders in tenant-ID order. + expect( + (await rows(w, 1)).reduce( + (counts, row) => { + const i = tenants.indexOf(row.tenantId as (typeof tenants)[number]); + counts[i]!++; + return counts; + }, + [0, 0], + ), + ).toEqual([250, 100]); + }, + "a new generation registered as {mode}": async (w) => { + const mark = await backendOf(w).admin.logMark(); + w.batchStart = mark.cursorMs; + const id = await backendOf(w).admin.run( + getFunctionName(internal.rebuild.startGeneration), + { readModel, projectionVersion: 1, batchSize: 1, operator }, + ); + if (typeof id !== "string") throw new Error("Expected a generation ID"); + w.generation2 = id; + w.initial = await entry(w); + w.oldFence = w.initial.generation.fence; + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: online generation 2 uses version 1, batch size 1 and the stated operator. + expect(w.initial).toMatchObject({ + generation: { + generation: 2, + state: "building", + projectionVersion: 1, + pauseRequired: false, + startedBy: operator, + }, + progress: { batchSize: 1, pass: "backfill" }, + }); + }, + "live commands {liveCommands}": async (w) => { + startLive(w); + await waitForGeneration( + backendOf(w), + readModel, + id2(w), + () => w.liveCounts.every((count) => count >= 1), + (entry) => observe(w, entry), + ); + }, + "the backfill is interrupted after {batches} batches and then {resumption}": + async (w, { batches }) => { + await waitForGeneration( + backendOf(w), + readModel, + id2(w), + ({ progress }) => progress.batchesDone >= batches, + (entry) => observe(w, entry), + ); + await backendOf(w).admin.run( + getFunctionName(internal.rebuild.interruptGeneration), + { generationId: id2(w), operator: "interrupt operator" }, + ); + await stopLive(w); + w.interrupted = await entry(w); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: the interrupt lands after at least three and below 350 batches, with a tenant cursor. + expect(w.interrupted.progress.batchesDone).toBeGreaterThanOrEqual( + batches, + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: backfill is interrupted before 350 batches. + expect(w.interrupted.progress.batchesDone).toBeLessThan(350); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: the cursor names a tenant. + expect(tenants).toContain(w.interrupted.progress.cursor?.tenantId); + // rebuild.sdp.md:130: the exact interrupt reason is retained at the checkpoint. + expect(w.interrupted.progress.lastError).toBe( + "interrupted by interrupt operator", + ); + await sleep(150); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: the interrupted checkpoint stops advancing. + expect((await entry(w)).progress).toEqual(w.interrupted.progress); + const target = await rows(w, 2); + w.missingOrder = Array.from({ length: 100 }, (_, i) => seeded(i)).find( + (key) => + !target.some( + (row) => row.tenantId === tenants[1] && row.key === key, + ), + ); + w.existingOrder = target.find( + (row) => + row.tenantId === tenants[0] && + row.status === "placed" && + String(row.key).startsWith("seed-"), + )?.key as string | undefined; + await cancel( + w, + 1, + required( + w.missingOrder, + "a second-tenant order backfill has not written", + ), + ); + await cancel( + w, + 0, + required( + w.existingOrder, + "a first-tenant row read before cancellation", + ), + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:34: an existing building row is updated immediately by the live cancel. + expect( + (await rows(w, 2)).find( + (row) => row.tenantId === tenants[0] && row.key === w.existingOrder, + )?.status, + ).toBe("cancelled"); + // rebuild.sdp.md:69: an existing subject without a building row remains absent until a batch writes it. + expect( + (await rows(w, 2)).find( + (row) => row.tenantId === tenants[1] && row.key === w.missingOrder, + ), + ).toBeUndefined(); + const checkpoint = (await entry(w)).progress; + await backendOf(w).admin.run( + getFunctionName(internal.rebuild.backfillBatch), + { + generationId: id2(w), + fence: required(w.oldFence, "the replaced fence"), + }, + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: a queued batch under the replaced fence cannot checkpoint. + expect((await entry(w)).progress).toEqual(checkpoint); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: interrupt changes only fence and operator change fields. + expect( + without(w.interrupted.generation, [ + "fence", + "changedAt", + "changedBy", + ]), + ).toEqual( + without(required(w.initial, "the start row").generation, [ + "fence", + "changedAt", + "changedBy", + ]), + ); + // rebuild.sdp.md:130: the interrupt replaces the fence and records the operator. + expect(w.interrupted.generation).toMatchObject({ + fence: required(w.oldFence, "the original fence") + 1, + changedBy: "interrupt operator", + }); + await backendOf(w).admin.run( + getFunctionName(internal.rebuild.resumeGeneration), + { generationId: id2(w), operator: "resume operator" }, + ); + w.resumed = await entry(w); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: resume changes only fence and operator change fields. + expect( + without(w.resumed.generation, ["fence", "changedAt", "changedBy"]), + ).toEqual( + without(w.interrupted.generation, [ + "fence", + "changedAt", + "changedBy", + ]), + ); + // rebuild.sdp.md:131: resume replaces the fence and keeps the batch size. + expect(w.resumed).toMatchObject({ + generation: { + fence: w.interrupted.generation.fence + 1, + changedBy: "resume operator", + }, + progress: { batchSize: 1 }, + }); + // rebuild.sdp.md:131: resume clears the interrupt error. + expect(w.resumed.progress.lastError).toBeUndefined(); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: each operator change records a new time. + expect(Number(w.interrupted.generation.changedAt)).toBeGreaterThan( + Number(required(w.initial, "the start row").generation.changedAt), + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: resume records its own time. + expect(Number(w.resumed.generation.changedAt)).toBeGreaterThan( + Number(w.interrupted.generation.changedAt), + ); + const resumedAt = w.cursors.length; + startLive(w); + w.verified = await waitForGeneration( + backendOf(w), + readModel, + id2(w), + undefined, + (entry) => observe(w, entry), + ); + await stopLive(w); + w.batchEnd = (await backendOf(w).admin.logMark()).cursorMs; + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: the backfill cursor visits the first tenant before the second. + expect(w.cursors.indexOf(tenants[0])).toBeGreaterThanOrEqual(0); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: the cursor crosses to the second tenant after resume. + expect(w.cursors.indexOf(tenants[1])).toBeGreaterThan( + w.cursors.indexOf(tenants[0]), + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: after the resume the cursor names the second tenant. + expect(w.cursors.slice(resumedAt)).toContain(tenants[1]); + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: after the resume it never returns to the first tenant once it left it. + const afterResume = w.cursors.slice(resumedAt); + expect( + afterResume.slice(afterResume.indexOf(tenants[1])), + ).not.toContain(tenants[0]); + // rebuild.online-rebuild-interrupt-resume.sdp.md:34: the second tenant's interrupted cancel survives whichever pass wrote its row. + expect( + (await rows(w, 2)).find( + (row) => row.tenantId === tenants[1] && row.key === w.missingOrder, + )?.status, + ).toBe("cancelled"); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: batches alter only state on the generation row. + expect(without(w.verified.generation, ["state"])).toEqual( + without(w.resumed.generation, ["state"]), + ); + }, + "no row of the new generation holds data older than its source stream version": + (w) => assertCoverage(w, 2), + "the new generation {coverage}": async (w) => { + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: both passes have ended before cutover. + expect(required(w.verified, "verified generation")).toMatchObject({ + generation: { state: "verified" }, + progress: { pass: "idle", cursor: null }, + }); + }, + "no command or external effect ran during the rebuild": async (w) => { + await assertJournals(w); + await assertLog(w); + }, + "the active generation {cutover}": async (w) => { + await subscribe(w); + await switchAndObserve(w, id2(w)); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: the first switch retires generation 1 and activates generation 2. + expect( + (await generations(backendOf(w), readModel)).map(({ generation }) => [ + generation.generation, + generation.state, + ]), + ).toEqual([ + [2, "active"], + [1, "retired"], + ]); + await place(w, 0, "after-switch"); + const old = w.watches[0]!.values.at(-1)!.page.find( + (row) => row.status === "placed" && row.key.startsWith("seed-"), + ); + const cancelled = String(required(old, "an earlier placed order").key); + await cancel(w, 0, cancelled); + await w.watches[0]!.until( + (page) => !page.page.some((row) => row.key === cancelled), + "the live writes before rollback", + ); + const retiredRows = await rows(w, 1); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37; rebuild.sdp.md:74: the retired generation was not written by the new order. + expect( + retiredRows.find( + (row) => row.tenantId === tenants[0] && row.key === "after-switch", + ), + ).toBeUndefined(); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37; rebuild.sdp.md:74: the retired row still needs its cancellation repaired. + expect( + retiredRows.find( + (row) => row.tenantId === tenants[0] && row.key === cancelled, + )?.status, + ).toBe("placed"); + const generation1 = required(w.generation1, "generation 1"); + const before = await entry(w, generation1); + const expectedBatches = await verifyBatchCount( + w, + before.progress.batchSize, + ); + const rollbackMark = await backendOf(w).admin.logMark(); + await backendOf(w).admin.run( + getFunctionName(internal.rebuild.rollbackGeneration), + { generationId: generation1, operator }, + ); + const verified = await waitForGeneration( + backendOf(w), + readModel, + generation1, + undefined, + () => drain(w), + ); + const completions = await backendOf(w).admin.completionsSince( + rollbackMark, + (seen) => + seen.some((record) => record.identifier === "rebuild:verifyBatch"), + ); + for (const completion of completions) + w.log.set(completion.executionId, completion as Execution); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: rollback runs only verify batches, never a backfill pass. + expect( + completions.filter( + (record) => record.identifier === "rebuild:backfillBatch", + ), + ).toEqual([]); + // rebuild.sdp.md:134: rollback retains generation 1's batch size and cumulative counts. + expect(verified.progress.batchSize).toBe(before.progress.batchSize); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: every checkpoint in rollback is one committed verify batch. + expect(verified.progress.batchesDone - before.progress.batchesDone).toBe( + completions.filter( + (record) => + record.identifier === "rebuild:verifyBatch" && + !(record as Execution).willRetry && + record.error === null, + ).length, + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37; rebuild.sdp.md:127: exactly one verify pass, at generation 1's retained size. + expect(verified.progress.batchesDone - before.progress.batchesDone).toBe( + expectedBatches, + ); + await assertCoverage(w, 1); + const repaired = await rows(w, 1); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: rollback repairs the new order before activation. + expect(repaired).toContainEqual( + expect.objectContaining({ + tenantId: tenants[0], + key: "after-switch", + status: "placed", + }), + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: rollback repairs the cancellation before activation. + expect(repaired).toContainEqual( + expect.objectContaining({ + tenantId: tenants[0], + key: cancelled, + status: "cancelled", + }), + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: the repaired generation is verified, while generation 2 still serves queries. + expect( + (await generations(backendOf(w), readModel)).map(({ generation }) => [ + generation.generation, + generation.state, + ]), + ).toEqual([ + [2, "active"], + [1, "verified"], + ]); + await switchAndObserve(w, generation1); + // rebuild.online-rebuild-interrupt-resume.sdp.md:37: the return switch activates generation 1 and retires generation 2. + expect( + (await generations(backendOf(w), readModel)).map(({ generation }) => [ + generation.generation, + generation.state, + ]), + ).toEqual([ + [2, "retired"], + [1, "active"], + ]); + }, + "writes {writes}": async (w) => { + // rebuild.online-rebuild-interrupt-resume.sdp.md:33,36: both ordinary callers placed and cancelled orders while batches ran. + expect(w.liveCounts.every((count) => count >= 2)).toBe(true); + await assertJournals(w); + await assertLog(w); + }, + }, +); From cac73fd785e64e796f433424966a877e31832441 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:13:34 +0200 Subject: [PATCH 084/155] build: 3b integration the compiled rebuild proof's refused arguments and optional batch size Each @ts-expect-error of a refused argument moves onto the property the validator refuses, since a directive binds only the next line. The batchSizeFor parameter tuple names requested as number | undefined, which is what an optional parameter gives under exactOptionalPropertyTypes. Co-Authored-By: Claude Opus 5.5 --- tests/types/rebuild.test-d.ts | 38 +++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/types/rebuild.test-d.ts b/tests/types/rebuild.test-d.ts index 655bd4d..ed5901a 100644 --- a/tests/types/rebuild.test-d.ts +++ b/tests/types/rebuild.test-d.ts @@ -101,10 +101,10 @@ test("compiled: example startGeneration arguments, visibility and return", () => projectionVersion: 2, batchSize: 1, }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(example.rebuild.startGeneration, { readModel: "documentTitle", operator: "operator", + // @ts-expect-error The entry accepts only the arguments its validator declares. projectionVersion: "2", }); // @ts-expect-error The return validator does not accept a boolean. @@ -165,10 +165,10 @@ test("compiled: example resumeGeneration arguments, visibility and return", () = generationId, operator: "operator", }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(example.rebuild.resumeGeneration, { generationId, operator: "operator", + // @ts-expect-error The entry accepts only the arguments its validator declares. batchSize: 2, }); // @ts-expect-error The return validator does not accept a boolean. @@ -208,10 +208,10 @@ test("compiled: example switchGeneration arguments, visibility and return", () = operator: "operator", rollbackPeriodMs: 0, }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(example.rebuild.switchGeneration, { generationId, operator: "operator", + // @ts-expect-error The entry accepts only the arguments its validator declares. rollbackPeriodMs: "0", }); // @ts-expect-error The return validator does not accept a boolean. @@ -242,9 +242,9 @@ test("compiled: example rollbackGeneration arguments, visibility and return", () generationId, operator: "operator", }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(example.rebuild.rollbackGeneration, { generationId, + // @ts-expect-error The entry accepts only the arguments its validator declares. operator: 1, }); // @ts-expect-error The return validator does not accept a boolean. @@ -313,10 +313,10 @@ test("compiled: example purgeGeneration arguments, visibility and return", () => generationId, operator: "operator", }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(example.rebuild.purgeGeneration, { generationId, operator: "operator", + // @ts-expect-error The entry accepts only the arguments its validator declares. batchSize: 1, }); // @ts-expect-error The return validator does not accept a boolean. @@ -347,9 +347,9 @@ test("compiled: example backfillBatch arguments, visibility and return", () => { generationId, fence: 1, }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(example.rebuild.backfillBatch, { generationId, + // @ts-expect-error The entry accepts only the arguments its validator declares. fence: "1", }); // @ts-expect-error The return validator does not accept a boolean. @@ -402,10 +402,10 @@ test("compiled: example purgeBatch arguments, visibility and return", () => { const ctx = {} as MutationCtx; const generationId = "generation" as Id<"generations">; void ctx.runMutation(example.rebuild.purgeBatch, { generationId, fence: 1 }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(example.rebuild.purgeBatch, { generationId, fence: 1, + // @ts-expect-error The entry accepts only the arguments its validator declares. batchSize: 1, }); // @ts-expect-error The return validator does not accept a boolean. @@ -448,9 +448,9 @@ test("compiled: example fillTenantsBatch arguments, visibility and return", () = const ctx = {} as MutationCtx; const generationId = "generation" as Id<"generations">; void ctx.runMutation(example.rebuild.fillTenantsBatch, { fence: 1 }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(example.rebuild.fillTenantsBatch, { fence: 1, + // @ts-expect-error The entry accepts only the arguments its validator declares. generationId, }); // @ts-expect-error The return validator does not accept a boolean. @@ -480,9 +480,9 @@ test("compiled: example getGenerations arguments, visibility and return", () => void ctx.runQuery(example.rebuild.getGenerations, { readModel: "documentTitle", }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runQuery(example.rebuild.getGenerations, { readModel: "documentTitle", + // @ts-expect-error The entry accepts only the arguments its validator declares. operator: "operator", }); // @ts-expect-error The return validator does not accept a boolean. @@ -524,10 +524,10 @@ test("compiled: fixture startGeneration arguments, visibility and return", () => projectionVersion: 2, batchSize: 1, }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(fixture.rebuild.startGeneration, { readModel: "documentTitle", operator: "operator", + // @ts-expect-error The entry accepts only the arguments its validator declares. projectionVersion: "2", }); // @ts-expect-error The return validator does not accept a boolean. @@ -588,10 +588,10 @@ test("compiled: fixture resumeGeneration arguments, visibility and return", () = generationId, operator: "operator", }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(fixture.rebuild.resumeGeneration, { generationId, operator: "operator", + // @ts-expect-error The entry accepts only the arguments its validator declares. batchSize: 2, }); // @ts-expect-error The return validator does not accept a boolean. @@ -631,10 +631,10 @@ test("compiled: fixture switchGeneration arguments, visibility and return", () = operator: "operator", rollbackPeriodMs: 0, }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(fixture.rebuild.switchGeneration, { generationId, operator: "operator", + // @ts-expect-error The entry accepts only the arguments its validator declares. rollbackPeriodMs: "0", }); // @ts-expect-error The return validator does not accept a boolean. @@ -665,9 +665,9 @@ test("compiled: fixture rollbackGeneration arguments, visibility and return", () generationId, operator: "operator", }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(fixture.rebuild.rollbackGeneration, { generationId, + // @ts-expect-error The entry accepts only the arguments its validator declares. operator: 1, }); // @ts-expect-error The return validator does not accept a boolean. @@ -736,10 +736,10 @@ test("compiled: fixture purgeGeneration arguments, visibility and return", () => generationId, operator: "operator", }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(fixture.rebuild.purgeGeneration, { generationId, operator: "operator", + // @ts-expect-error The entry accepts only the arguments its validator declares. batchSize: 1, }); // @ts-expect-error The return validator does not accept a boolean. @@ -770,9 +770,9 @@ test("compiled: fixture backfillBatch arguments, visibility and return", () => { generationId, fence: 1, }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(fixture.rebuild.backfillBatch, { generationId, + // @ts-expect-error The entry accepts only the arguments its validator declares. fence: "1", }); // @ts-expect-error The return validator does not accept a boolean. @@ -825,10 +825,10 @@ test("compiled: fixture purgeBatch arguments, visibility and return", () => { const ctx = {} as MutationCtx; const generationId = "generation" as Id<"generations">; void ctx.runMutation(fixture.rebuild.purgeBatch, { generationId, fence: 1 }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(fixture.rebuild.purgeBatch, { generationId, fence: 1, + // @ts-expect-error The entry accepts only the arguments its validator declares. batchSize: 1, }); // @ts-expect-error The return validator does not accept a boolean. @@ -871,9 +871,9 @@ test("compiled: fixture fillTenantsBatch arguments, visibility and return", () = const ctx = {} as MutationCtx; const generationId = "generation" as Id<"generations">; void ctx.runMutation(fixture.rebuild.fillTenantsBatch, { fence: 1 }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runMutation(fixture.rebuild.fillTenantsBatch, { fence: 1, + // @ts-expect-error The entry accepts only the arguments its validator declares. generationId, }); // @ts-expect-error The return validator does not accept a boolean. @@ -903,9 +903,9 @@ test("compiled: fixture getGenerations arguments, visibility and return", () => void ctx.runQuery(fixture.rebuild.getGenerations, { readModel: "documentTitle", }); - // @ts-expect-error The entry accepts only the arguments its validator declares. void ctx.runQuery(fixture.rebuild.getGenerations, { readModel: "documentTitle", + // @ts-expect-error The entry accepts only the arguments its validator declares. operator: "operator", }); // @ts-expect-error The return validator does not accept a boolean. @@ -950,7 +950,7 @@ test("compiled: rebuild configuration and helpers have the pinned signatures", ( }>(); type AnyModel = RebuildTarget["readModel"]; expectTypeOf().parameters.toEqualTypeOf< - [readModel: AnyModel, requested?: number] + [readModel: AnyModel, requested?: number | undefined] >(); expectTypeOf().parameters.toEqualTypeOf< [readModel: AnyModel] From 194354654e87a5a0cf47691b55b53cfbc89b8a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:17:00 +0200 Subject: [PATCH 085/155] build: 3b integration the three setups still on the singular projection and the old generation row The order summary's and the document title's tests read projections, the title's over both versions, and the gate read's two generations are inserted in the slimmer registry row. Co-Authored-By: Claude Opus 5.5 --- tests/pure/example-domain.test.ts | 6 ++--- tests/pure/fixture-faults.test.ts | 34 +++++++++++++++-------------- tests/simulator/write-pause.test.ts | 10 +++------ 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/tests/pure/example-domain.test.ts b/tests/pure/example-domain.test.ts index 3906025..c081bb0 100644 --- a/tests/pure/example-domain.test.ts +++ b/tests/pure/example-domain.test.ts @@ -289,9 +289,9 @@ test("pure: a placed order's DTO carries its lines, total, time and stream versi test("pure: the order summary projects the order's DTO to its status, line count, total and time, keyed by the order ID", () => { const dto = orderStream.toDto(placed(), meta("order-1")) as Parameters< - typeof orderSummary.projection.project + (typeof orderSummary.projections)[0]["project"] >[1]; - const { projection } = orderSummary; + const [projection] = orderSummary.projections; expect(projection.keyOf("t-1", dto)).toBe("order-1"); const row = projection.project("t-1", dto, [dto.version]); expect(row).toStrictEqual({ @@ -315,7 +315,7 @@ test("pure: the order summary projects the order's DTO to its status, line count name: "orderSummary", table: "orderSummaries", rowBudgetBytes: 16384, - projection: { version: 1 }, + projections: [{ version: 1 }], }); }); diff --git a/tests/pure/fixture-faults.test.ts b/tests/pure/fixture-faults.test.ts index c355b40..ca581dc 100644 --- a/tests/pure/fixture-faults.test.ts +++ b/tests/pure/fixture-faults.test.ts @@ -89,20 +89,22 @@ test("the copy's toDto throws for the yard's fault title only, naming the depot }); test("the document title projection throws for its fault title only", () => { - const { projection } = documentTitle; - expect( - projection.keyOf("t-1", { documentId: "doc-1", title: "Report" }), - ).toBe("doc-1"); - expect( - projection.project("t-1", { documentId: "doc-1", title: "Report" }, []), - ).toEqual({ documentId: "doc-1", title: "Report" }); - expect(() => - projection.project( - "t-1", - { documentId: "doc-1", title: projectionFaultTitle }, - [], - ), - ).toThrow( - `Fault injected: ${projectionFaultTitle}, projecting document doc-1`, - ); + // native-harness.sdp.md:90: both versions throw for the fault title. + for (const projection of documentTitle.projections) { + expect( + projection.keyOf("t-1", { documentId: "doc-1", title: "Report" }), + ).toBe("doc-1"); + expect( + projection.project("t-1", { documentId: "doc-1", title: "Report" }, []), + ).toEqual({ documentId: "doc-1", title: "Report" }); + expect(() => + projection.project( + "t-1", + { documentId: "doc-1", title: projectionFaultTitle }, + [], + ), + ).toThrow( + `Fault injected: ${projectionFaultTitle}, projecting document doc-1`, + ); + } }); diff --git a/tests/simulator/write-pause.test.ts b/tests/simulator/write-pause.test.ts index 08047e8..d41ff4a 100644 --- a/tests/simulator/write-pause.test.ts +++ b/tests/simulator/write-pause.test.ts @@ -159,15 +159,11 @@ describe("the gate read", () => { projectionVersion: 1, state: "building" as const, pauseRequired: true, - batchSize: 1, fence: 0, - batchesDone: 0, - rowsWritten: 0, - rowsSkipped: 0, - misses: 0, startedAt: 0, - startedBy: { kind: "operator" as const, id: "ops-1" }, - updatedAt: 0, + startedBy: "ops-1", + changedAt: 0, + changedBy: "ops-1", }; return [ await ctx.db.insert("generations", { ...row, generation: 1 }), From 32b742055666be7f5133c8ccdc569affe9c41067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:17:00 +0200 Subject: [PATCH 086/155] design: 3b integration the online rebuild example names its composition Its native test runs on the production composition, and a native example with a verifier names the composition it runs on. Co-Authored-By: Claude Opus 5.5 --- .../application/rebuild.online-rebuild-interrupt-resume.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md index 91f7218..45b44dc 100644 --- a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md +++ b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md @@ -9,7 +9,7 @@ relations: --- # Rebuild a per-entity read model online while commands run; interrupt and resume -Sc L2-5 · native tier. +Sc L2-5 · native tier · production composition. ## Intent From a013ba4e98534d89f4c86a4262986ad09bf30734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:26:21 +0200 Subject: [PATCH 087/155] build: 3b integration the scheduled rows a command or the scheduler states added, apart from the install's A setup that installs a read model through its first rebuild leaves its batches' scheduled rows behind. The command examples count the rows added since a read taken before the command, and Probe 7 counts the scheduler states' rows without the rebuild batches. Co-Authored-By: Claude Opus 5.5 --- tests/native/first-experiment-steps.ts | 10 ++++--- ...committed-state-read-through-query.test.ts | 11 ++++++-- ...ead-models.committed-state-visible.test.ts | 11 ++++++-- tests/native/scheduled-rows.ts | 8 ++++++ tests/native/scheduler-composition.ts | 26 ++++++++++--------- tests/native/scheduler-restore.ts | 14 ++-------- 6 files changed, 49 insertions(+), 31 deletions(-) diff --git a/tests/native/first-experiment-steps.ts b/tests/native/first-experiment-steps.ts index ad18643..a8b1254 100644 --- a/tests/native/first-experiment-steps.ts +++ b/tests/native/first-experiment-steps.ts @@ -26,7 +26,7 @@ import type { CompletionRecord } from "../../harness/admin.js"; import type { Backend } from "../../harness/backend.js"; import { ordinaryClient } from "../../harness/clients.js"; import { measure, productionBackend, required } from "../../harness/native.js"; -import { scheduledRows } from "./scheduled-rows.js"; +import { scheduledRows, scheduledSince } from "./scheduled-rows.js"; import { utf8Length, type MutationCtx } from "../../src/command/index.js"; export const tenantId = "t-1"; export const subject = "user-1"; @@ -214,7 +214,7 @@ export async function place( summaryKeys: summaries.page.map((row) => row.key), }; } -// The scheduled-function tables of the parent and both contexts, read after the commands: a job +// The scheduled-function rows of the parent and both contexts that the commands added: a job // scheduled with any delay is a row there. const scheduledAfter = new WeakMap< ExperimentWorld, @@ -230,6 +230,7 @@ export async function placeOrderRuns( if (run !== "the PlaceOrder use case") throw new Error(`These examples run the PlaceOrder use case, not ${run}`); const size = required(world.size, "the order's size"); + const before = await scheduledRows(required(world.backend, "the backend")); if (linesOf[size] > 1) world.reference = await place(world, "order-ref", orderLines(1, "ref")); world.placed = await place( @@ -239,7 +240,10 @@ export async function placeOrderRuns( ); scheduledAfter.set( world, - await scheduledRows(required(world.backend, "the backend")), + scheduledSince( + before, + await scheduledRows(required(world.backend, "the backend")), + ), ); } const sent = (world: ExperimentWorld): Placed[] => diff --git a/tests/native/read-models.committed-state-read-through-query.test.ts b/tests/native/read-models.committed-state-read-through-query.test.ts index 18b29bd..7d70cbe 100644 --- a/tests/native/read-models.committed-state-read-through-query.test.ts +++ b/tests/native/read-models.committed-state-read-through-query.test.ts @@ -10,7 +10,7 @@ import { committedStateReadThroughQueryContract as contract } from "../../genera import { api } from "../../example/convex/_generated/api.js"; import type { LogMark } from "../../harness/admin.js"; import { required } from "../../harness/native.js"; -import { scheduledRows } from "./scheduled-rows.js"; +import { scheduledRows, scheduledSince } from "./scheduled-rows.js"; import { line, orderWorld, @@ -34,6 +34,7 @@ type Response = FunctionReturnType; interface World { order?: OrderWorld; mark?: LogMark; + scheduledBefore?: Awaited>; response?: Response; } const orderId = "order-1"; @@ -64,6 +65,7 @@ bindExample(contract, (): World => ({}), { if (action !== "the use case commits one successful command") throw new Error(`This test binds one successful command`); const { backend, client } = required(world.order, "the backend"); + world.scheduledBefore = await scheduledRows(backend); world.mark = await backend.admin.logMark(); world.response = await client.mutation(api.ordering.placeOrder, { tenantId, @@ -121,7 +123,12 @@ bindExample(contract, (): World => ({}), { ).toHaveLength(workers); // The scheduled-function tables of the parent and both contexts also hold a job scheduled with // a delay past the window. - expect(await scheduledRows(backend)).toHaveLength(workers); + expect( + scheduledSince( + required(world.scheduledBefore, "the scheduled rows before"), + await scheduledRows(backend), + ), + ).toHaveLength(workers); expect( records.find((entry) => entry.identifier === placeOrderIdentifier), ).toMatchObject({ udfType: "Mutation", caller: "HttpApi", error: null }); diff --git a/tests/native/read-models.committed-state-visible.test.ts b/tests/native/read-models.committed-state-visible.test.ts index f78189a..d6bbb89 100644 --- a/tests/native/read-models.committed-state-visible.test.ts +++ b/tests/native/read-models.committed-state-visible.test.ts @@ -15,7 +15,7 @@ import { type QueryWatch, } from "../../harness/clients.js"; import { required } from "../../harness/native.js"; -import { scheduledRows } from "./scheduled-rows.js"; +import { scheduledRows, scheduledSince } from "./scheduled-rows.js"; import { line, orderWorld, @@ -37,6 +37,7 @@ interface World { order?: OrderWorld; summaries?: QueryWatch; mark?: LogMark; + scheduledBefore?: Awaited>; response?: Response; } const orderId = "order-1"; @@ -81,6 +82,7 @@ bindExample(contract, (): World => ({}), { if (action !== "the use case commits one successful command") throw new Error(`This test binds one successful command`); const { backend, client } = required(world.order, "the backend"); + world.scheduledBefore = await scheduledRows(backend); world.mark = await backend.admin.logMark(); world.response = await client.mutation(api.ordering.placeOrder, { tenantId, @@ -147,7 +149,12 @@ bindExample(contract, (): World => ({}), { ).toHaveLength(workers); // The scheduled-function tables of the parent and both contexts also hold a job scheduled with // a delay past the window. - expect(await scheduledRows(backend)).toHaveLength(workers); + expect( + scheduledSince( + required(world.scheduledBefore, "the scheduled rows before"), + await scheduledRows(backend), + ), + ).toHaveLength(workers); expect( records.find((entry) => entry.identifier === "ordering:placeOrder"), ).toMatchObject({ udfType: "Mutation", caller: "HttpApi", error: null }); diff --git a/tests/native/scheduled-rows.ts b/tests/native/scheduled-rows.ts index a224fb1..398e74c 100644 --- a/tests/native/scheduled-rows.ts +++ b/tests/native/scheduled-rows.ts @@ -11,3 +11,11 @@ export async function scheduledRows(backend: Backend) { ); return rows.flat(); } +// The rows of a later read that an earlier read did not hold: what was scheduled between the two. +// A setup that installs a read model through its first rebuild leaves its batches' rows behind. +export function scheduledSince( + before: Awaited>, + after: Awaited>, +) { + return after.filter((row) => !before.some((prior) => prior._id === row._id)); +} diff --git a/tests/native/scheduler-composition.ts b/tests/native/scheduler-composition.ts index edd8b5f..d1f6a2e 100644 --- a/tests/native/scheduler-composition.ts +++ b/tests/native/scheduler-composition.ts @@ -62,6 +62,19 @@ export async function schedulingBackend() { expect(backend.facts().composition).toBeNull(); return { backend, directory }; } +// The kinds of the scheduler states' rows in one scope. A setup that installs a read model through +// its first rebuild leaves its batches' rows in the parent; they are the setup's, not the states'. +export async function stateKinds( + backend: Backend, + component: (typeof scopes)[number], +) { + return ( + await backend.admin.readTable("_scheduled_functions", options(component)) + ) + .filter((row) => !String(row.name).startsWith("rebuild")) + .map((row) => (row.state as { kind: string }).kind) + .sort(); +} export async function initialize( backend: Backend, due: number, @@ -81,18 +94,7 @@ export async function initialize( } for (const component of scopes) await expect - .poll( - async () => - ( - await backend.admin.readTable( - "_scheduled_functions", - options(component), - ) - ) - .map((row) => (row.state as { kind: string }).kind) - .sort(), - { timeout: 15000 }, - ) + .poll(() => stateKinds(backend, component), { timeout: 15000 }) .toEqual(["canceled", "failed", "inProgress", "pending", "success"]); } export async function schedulerRows(backend: Backend) { diff --git a/tests/native/scheduler-restore.ts b/tests/native/scheduler-restore.ts index 99e85c2..42f2dc9 100644 --- a/tests/native/scheduler-restore.ts +++ b/tests/native/scheduler-restore.ts @@ -8,6 +8,7 @@ import { scopes, options, record, + stateKinds, } from "./scheduler-composition.js"; export async function freshImport(backend: Backend, directory: string) { const exported = await exportData(backend, directory, Date.now() + 3600000); @@ -96,18 +97,7 @@ export async function inPlaceImport(backend: Backend, directory: string) { } for (const component of scopes) await expect - .poll( - async () => - ( - await backend.admin.readTable( - "_scheduled_functions", - options(component), - ) - ) - .map((row) => (row.state as { kind: string }).kind) - .sort(), - { timeout: 15000 }, - ) + .poll(() => stateKinds(backend, component), { timeout: 15000 }) .toEqual([ "canceled", "canceled", From e377f588935d02bd91bdb652101773f29f02cbd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:46:16 +0200 Subject: [PATCH 088/155] design: tests are the proof; the record files go The owner, 2026-10-03: "Tests should be all the proofs we need." and "We need to weed out all examples from the codebase and do all the suggested things." The repository's gates, run by CI on every push, are the proof. No agent writes a record file, a proof log or a per-number command list, and no builder or fold-in shows a test failing under a mutation; mutation testing stays the reviewer's, in its own report. A GPT job's report stays in its job directory under ~/.codex-jobs, and a Claude agent answers in the thread in at most fifteen lines. SESSIONS.md: the fourth rule of the 2026-09-30 stop, the role table, slice steps 3, 4 and 6, the hand-over seam and note. AGENTS.md: the rule for every reader, and the check report out of the unit folder. advisors/README.md: the check report stays in its job directory. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 ++- design/SESSIONS.md | 16 ++++++++-------- design/advisors/README.md | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8426c3f..b27e4bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -16,6 +16,7 @@ The Convex transactional domain platform. `docs/` holds the inputs. `design/` ho - Before you write or edit a Spec, read `design/PLAN.md` section 6 and `node_modules/@libar-dev/software-delivery-protocol/.agents/skills/sdp-authoring/SKILL.md`. - `python3 design/tools/check.py` prints `OK` when the Specs, the README and the ledger agree. Run it before you say they do. - Name the tier behind a claim about behavior: compiled, pure test, `convex-test`, or native backend. +- Tests and CI are the proof. No agent writes a record file, a proof log or a per-number command list; a job answers in fifteen lines and the run log takes one line. The owner, 2026-10-03: "Tests should be all the proofs we need." - The earlier platform, in the `convex-event-sourcing` repository, is evidence about Convex and about what failed. Take a fact from it with its date and versions. Take none of its designs or working habits. - `design/reviews/r3-*.jsonl` are records of a finished review round and keep their text. - Prose uses plain words, sentence-case headings and no em dashes. @@ -25,7 +26,7 @@ The Convex transactional domain platform. `docs/` holds the inputs. `design/` ho - `design/ROADMAP.md` is the aggregated roadmap: the next units in rounds, what each needs first and shows, what is deferred and why. The owner rules on it; the main thread keeps it current at each close. The slices of `design/STATE.md` keep their names, and the roadmap orders the units inside and between them. - Four advisors, `advisor-convex`, `advisor-domain`, `advisor-operator` and `advisor-product`, are Fable agents defined under `.claude/agents/`. Each is launched with a task file under `design/advisors/`: `task-sort.md` sorts the open decisions of its lens, `task-fork.md` writes one owner decision as a fork, `task-roadmap.md` proposes the milestones of one subject. The four run at once, with disjoint subjects, and the main thread merges or cuts. - Every lean, fork or memo is checked by a `gpt-6.1-sol` job of the other family before the owner sees it, on a copy that carries one planted false citation. `design/advisors/README.md` says how each run goes and what the first runs showed; `design/advisors/protocol.md` binds every sentence an advisor writes. -- The memos, briefs, plants and check reports of a run live outside the repository, in the unit folder the project notes name. +- The memos, briefs and plants of a run live outside the repository, in the unit folder the project notes name. A check job's report stays in its job directory. ## The language diff --git a/design/SESSIONS.md b/design/SESSIONS.md index c0f6a55..b53376d 100644 --- a/design/SESSIONS.md +++ b/design/SESSIONS.md @@ -2,7 +2,7 @@ A session is one Claude Code conversation with Fable 5.1 as the main thread. It does one unit of work and leaves the Specs, the code, `README.md`, the ledger and `STATE.md` agreeing. -The main thread coordinates. It rules on every finding, approves every Spec sentence, and owns the check, the push and the merge. It delegates reading, checking, building and the writing of Spec text from its rulings, and it keeps its own context small: a job answers it in a few lines and leaves its report in a file. The design is finished by building it: the doc says detail follows the build, and three review rounds on paper did not converge. +The main thread coordinates. It rules on every finding, approves every Spec sentence, and owns the check, the push and the merge. It delegates reading, checking, building and the writing of Spec text from its rulings, and it keeps its own context small: a job answers it in at most fifteen lines, and the repository's gates, which CI runs on every push, are the proof of what it built. The design is finished by building it: the doc says detail follows the build, and three review rounds on paper did not converge. ## What wins when sources disagree @@ -28,7 +28,7 @@ The owner ruled on 2026-10-01 that the session takes tactical decisions on its o | The design of a solution's key abstractions before an implementer writes it: interfaces as stubs, names, shapes, bounds, proof obligations, and exact Spec text as a patch proven in a scratch copy | Claude agent, Fable 5.1 | default | | Building from a Spec, integrating parallel builders and folding in ruled findings | Claude agent, Opus 5.5, usually inside one Workflow run | default | | Building a correction or a bounded package from exact Spec text, in a worktree of its own | `gpt-6.1-sol` | medium | -| The proof before the build: tests written from the ruled Spec text alone, shown to fail before the fix, with an attack on the ruled behavior | `gpt-6-astra` | high | +| The proof before the build: tests written from the ruled Spec text alone, with an attack on the ruled behavior | `gpt-6-astra` | high | | Corpus bookkeeping from exact rulings: ledger records, register rows, recomputed numbers, coverage tables, each as a draft with an apply script proven on a scratch copy | `gpt-6.1-sol` | medium | | Read-only scout before a unit; measurement and controlled comparisons on the native backend; the behavior review lane, which runs things | `gpt-6-astra` | high | | The mechanical review lane over what Claude models wrote: every changed line, test, number and pinned value | `gpt-6.1-sol` | medium | @@ -48,7 +48,7 @@ Four rules from the stop of 2026-09-30, when ten parallel Fable agents hit the u - One unit per session. - One write lane per tree. A second writer gets its own git worktree and files the first does not touch, and an integrator agent or the main thread merges its branch. Two writers share one tree only when the two briefs split the files and each names the other's files as forbidden, and then nothing is committed or checked until both are done. - The ledger is written ahead: each finding and each status before the next begins. -- Every delegated job's report is saved to a file the moment it arrives. +- A GPT job's report stays in its job directory and a Claude agent answers in the thread. The rule to save every report to a file is gone, because a job's report now survives in `~/.codex-jobs`. ## Open @@ -64,10 +64,10 @@ One slice of the build, as `STATE.md` cuts them. `python3 design/tools/check.py 1. **Rule first, in one page.** The main thread writes its rulings on the slice's paper decisions and its thin cut (what the slice builds and what it leaves out) into one rulings file. A platform decision gets its provisional reading there and a line in the owner queue. This file goes into every builder's prompt. The main thread does not read the Specs of the slice to do this: `STATE.md` names the decisions. 2. **Design inside the workflow, where the slice has design to do.** The stages before the code are stages of the same kind of run: a design review of the Specs the slice builds, against the rulings and the doc; the Spec updates that review leads to, as proposed `## Design` entries and rules; an implementation plan that names the packages, their files and their order. Fable agents design; Opus agents write the Spec text. Two rules of the owner from slice S2 shape these stages. One Fable agent decides each thing: the designers get disjoint subjects, split by subject and not only by file, a need from another's subject is a one-line request to the main thread, and no later Fable agent judges again what an earlier one decided. And a run is mostly producers: after the designers, several Opus writers put the text into disjoint Spec directories and a closer commits, with no advisor and no planner agent, because the main thread rules on the designers' short lists and cuts the packages itself. The designers are a short run of their own, and the Spec writers are the first stage of the build run. Size it to the slice. The first version of this corpus was written by one workflow with dozens of Fable agents, and the owner called the result great and the size excessive. Slice S2 had two Fable designers and thirteen Opus agents. Slice S1 needed no designer, because its Specs already pinned the signatures. -3. **Build as a workflow.** One Workflow run of Opus agents builds the slice from the Specs' `## Design` entries: packages that share files run in sequence in the main tree, each committing its own package as unreviewed work; packages that do not, such as groups of scenario tests, run in parallel, one git worktree and one branch each; one integrator merges, runs every tier including the whole native tier on a clean tree, and writes one list of what needs a ruling. Each agent returns a short structured answer and writes its long report to a file. An agent builds the nearest thing that compiles where a Spec does not, and lists it; it never edits a Spec outside what its prompt allows. Scenario agents may rebind their own example Specs and add the sibling examples the ledger asks for, and they quote every changed line. -4. **Scouts are optional and run beside the build.** A read-only GPT job that compiles the pinned declarations, checks the Convex claims on the pinned backend, or inventories a finding's restatements costs the thread nothing if its report goes to the builders by path. The main thread reads a verdict line from it, never the report. +3. **Build as a workflow.** One Workflow run of Opus agents builds the slice from the Specs' `## Design` entries: packages that share files run in sequence in the main tree, each committing its own package as unreviewed work; packages that do not, such as groups of scenario tests, run in parallel, one git worktree and one branch each; one integrator merges, runs every tier including the whole native tier on a clean tree, and writes one list of what needs a ruling. Each agent returns a structured answer of at most fifteen lines and writes no report file. An agent builds the nearest thing that compiles where a Spec does not, and lists it; it never edits a Spec outside what its prompt allows. Scenario agents may rebind their own example Specs and add the sibling examples the ledger asks for, and they quote every changed line. +4. **Scouts are optional and run beside the build.** A read-only GPT job that compiles the pinned declarations, checks the Convex claims on the pinned backend, or inventories a finding's restatements costs the thread nothing if the builders get the path of its report in its job directory. The main thread reads a verdict line from it, never the report. 5. **Rule on the integrator's list.** Code wrong becomes one numbered fold-in for an Opus agent. Spec wrong or silent becomes a list of exact rulings that an Opus agent applies to the Specs and the ledger, the main thread approving them through the one review that follows. A Spec names no slice and says nothing about what is built so far: that lives in `STATE.md`, the ledger and the code anchors. -6. **One review of the slice, on frozen copies, beside the fold-in.** Astra at xhigh runs the whole branch against the doc's rows and the Specs and tries mutations. Sol at medium reads every changed line, split in halves when the diff is large. Each brief asks for a first section of at most fifteen one-line findings on a path a healthy run or an ordinary failure takes; the main thread reads that section and the titles, and rules by class. A reviewer runs recipe 4 on the changed files. A file it lists under `coverageUnknownFiles` is reviewed against the Spec that owns its directory, and an empty impact set is never read as nothing to review. +6. **One review of the slice, on frozen copies, beside the fold-in.** Astra at xhigh runs the whole branch against the doc's rows and the Specs and tries mutations, reported in its own report: mutation testing is the reviewer's technique, and no builder or fold-in is asked to show a test failing under one. Sol at medium reads every changed line, split in halves when the diff is large. Each brief asks for a first section of at most fifteen one-line findings on a path a healthy run or an ordinary failure takes; the main thread reads that section and the titles, and rules by class. A reviewer runs recipe 4 on the changed files. A file it lists under `coverageUnknownFiles` is reviewed against the Spec that owns its directory, and an empty impact set is never read as nothing to review. 7. **One fold-in, one read of the fold-in, then stop.** Tactical decision 19 in `STATE.md` says when: what is left goes to Leads. A design agent proposes `## Design` entries and gets a time bound of about half an hour. Anything it built while designing is handed to the builders as a patch. @@ -124,11 +124,11 @@ Consensus holds for Layers 0 to 2 when the first experiment passes on a native b ## Handing over inside a unit -A build slice is too long for one conversation. The main thread hands over to a fresh main thread at a seam, before its own context gets in the way: after a commit, with no delegated job running and every report saved. +A build slice is too long for one conversation. The main thread hands over to a fresh main thread at a seam, before its own context gets in the way: after a commit, with no delegated job running. 1. Reach the seam. Wait for or stop every delegated job. A Claude agent dies with the conversation and leaves no report. A GPT job survives it, and its report, events and thread id are in its directory under `~/.codex-jobs`. 2. Commit what is in the tree on the unit's branch, as work in progress if it is not reviewed, and say so in the message. -3. Write the handover note beside the unit's reports, in the folder the project notes name. It says what is committed, what each tier showed and at which commit, which findings are ruled and where the rulings are, what is drafted and not applied, the next steps in order, and what waits for the owner. It lists every instruction the owner gave during the conversation, in the owner's words. +3. Write the handover note beside the unit's rulings page, in the folder the project notes name. It says what is committed, which gates fail at the last commit, which findings are ruled and where the rulings are, what is drafted and not applied, the next steps in order, and what waits for the owner. It lists every instruction the owner gave during the conversation, in the owner's words. 4. Put a short "In progress" block at the top of `STATE.md` that names the unit, the seam and the note. The rest of `STATE.md` waits for the close. 5. Start the next main thread from this one, over cmux. Two panes of one workspace take turns: the thread that hands over goes idle, and the other pane carries the work until it hands back. `cmux identify --json` names this pane, and the owner or `cmux list-panes` names the other. Read the other pane with `cmux read-screen --surface ` and make sure of three things before prompting it. Claude Code is running there in the repository; if it is not, start it with `cmux send`. The conversation is clear, with an empty context in its status line; if it is not, send `/clear`. A cleared conversation is required: a main thread that inherits a summary of this one inherits its weight. The model is Fable 5.1 at high effort, as its banner and status line show. Then send the prompt as one paragraph with `cmux send --surface `, because a newline submits, submit it with `cmux send-key --surface enter`, and read the screen once more to see that it started. The prompt names the unit, the note, and the files to read first. 6. The next main thread reads `STATE.md` and the note, runs the check, and confirms with `git status` and `git log` that the tree is what the note says before it does anything else. A note is one page: what is committed, what runs, what is ruled and where, the next steps, and the owner's instructions in the owner's words. diff --git a/design/advisors/README.md b/design/advisors/README.md index 7beaec6..20d3316 100644 --- a/design/advisors/README.md +++ b/design/advisors/README.md @@ -25,7 +25,7 @@ A definition holds three paths and nothing else. A fifth advisor is one lens fil ## How the main thread runs a sort -One launch for each advisor. The four run at once, because each writes its own patch and nothing else. `` is where the unit keeps its reports, outside the repository. +One launch for each advisor. The four run at once, because each writes its own patch and nothing else. `` is where the unit keeps its patches and memos, outside the repository. A Claude agent: @@ -58,7 +58,7 @@ Agent(subagent_type: "advisor-domain", Then, in this order: 1. The memos are copied to `/check-copy/`, and the main thread plants one false citation in each copy and records the plants in `/plants.md`, as tactical decision 31 of `STATE.md` asks for a review copy. -2. One `gpt:codex` job, `gpt-6.1-sol` at medium with the fast tier, checks the copies: every cited claim held, not held or not checked, every unlabelled fact listed. Its brief is `/check-brief.md`, and its report is saved to `/check-report.md` the moment it arrives. +2. One `gpt:codex` job, `gpt-6.1-sol` at medium with the fast tier, checks the copies: every cited claim held, not held or not checked, every unlabelled fact listed. Its brief is `/check-brief.md`, and its report stays in its job directory under `~/.codex-jobs`. 3. While the check runs, the main thread writes `design/ROADMAP.md`: the cut as one table, the rounds, what is deferred with its trigger, where the advisors differed and what was ruled, the questions that are not rows yet, the findings beyond the roadmap, and the words the advisors needed that `CONTEXT.md` lacks. 4. The claims the check marked not held are corrected in the roadmap, or the advisor is continued with the check's findings where the memo must change. The plants caught go into `plants.md` and the project notes. 5. The unit closes as any unit does: the check prints `OK`, `STATE.md` points at the roadmap, the commit goes up through a pull request. From 0481e9c5f7ae36f0682544b35b8f0b7f68229ab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:46:15 +0200 Subject: [PATCH 089/155] design: 3b the interrupt's reruns, the pinned page across a switch, the left-behind tables and the rebuild's text rulings The L2-5 example counts the reruns of operator entries and batches on the progress row beside the generations count and asserts only that no live command or parent query conflicts there. A read-model list pages with paginator and moves the client's cursors into the active generation, over an index that ends with key. The left-behind record names the three tables 3b deploys. The rebuild Spec names the batch size's source, the purge cap as a function, the purge end's patch, the missing progress row, the empty answer for an undeclared read model and updatedAt at the purge start; the first-rebuild example names the park under a closed tenant gate. Co-Authored-By: Claude Opus 5.5 --- design/specs/application/first-experiment.sdp.md | 2 +- ...istry.first-rebuild-installs-read-model.sdp.md | 2 +- .../specs/application/projection-contract.sdp.md | 2 +- design/specs/application/read-models.sdp.md | 15 ++++++++++----- ...rebuild.online-rebuild-interrupt-resume.sdp.md | 2 +- design/specs/application/rebuild.sdp.md | 10 +++++----- ...stry.first-rebuild-installs-read-model.test.ts | 4 ++-- 7 files changed, 21 insertions(+), 16 deletions(-) diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index d24ee99..00cfc1c 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -110,7 +110,7 @@ The experiment is an application, a test suite and a report. The application is - captureCommits: top-level commits are terminal top-level mutation completion records with `willRetry` false and `error` null; a completion with `willRetry` true is no commit even when `error` is null, and an authorized duplicate may commit a read-only transaction with no new business effect (E-47, First experiment) - captureCalls: function calls count the command's top-level completion records, including executions the engine reran; component calls and their individual time are `not exposed` by the native backend log, and a separate pure test of the use case executor counts calls by context with canned outcomes at each size (E-47, F4, E-15) - captureDocuments: documents and bytes come only from each top-level completion record's `usageStats`; the record retains every execution and sums its recorded counters by backend request ID and by cell, separately for terminal executions and executions refused at commit, without deriving rolled-back writes or a write set from stored rows; a field the backend does not expose is `not exposed` (E-47, First experiment, E-15) -- captureRowsLeftBehind: rows left behind are the documents a run creates that are not state, events, receipts or read-model rows; record their before, after and created counts by table, separately from setup, including markers, registry, gate and audit rows, and say `not deployed` for a table absent from the deployed schema (E-47, First experiment, D19) +- captureRowsLeftBehind: rows left behind are the documents a run creates that are not state, events, receipts or read-model rows; record their before, after and created counts by table, separately from setup, including the markers of `projectionMarkers`, the registry's `generations`, `generationProgress` and `tenantFill`, the gate and the audit rows, and say `not deployed` for a table absent from the deployed schema (E-47, First experiment, D19) - captureRetries: one completion with `willRetry` true and an `occInfo` is one execution refused at commit; its rerun has another completion with the same backend request ID; record these executions, the tables named by `occInfo` with the component that holds each, the engine's own `retryCount` of each refused execution, which numbers them from zero, each command's terminal record and final outcomes separately, and count terminal optimistic-concurrency failures as commands the engine failed; the variable number of reruns is an observation, never the difference between two cells (E-47, F1) - datasetSizes: 1 line, 10 lines and 100 lines, the maximum, each against a fresh disposable backend whose stock items are created through `ReceiveStock` (E-46, Sc L2-3, OQ3) - contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with stock for one, so that one is applied and the rest are rejected for short stock or failed by the engine after its bounded retries, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry; the shared stock item is the first line of every order and each other line is on a stock item of that command alone, received once with quantity two, and no caller retries during the cell, while the caller may repeat a command the engine failed under its same request key (First experiment, Sc L1-11, F1, E-5, E-47) diff --git a/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md b/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md index debbdb6..01045b3 100644 --- a/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md +++ b/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md @@ -27,6 +27,6 @@ And a second start while generation 1 is being built is {second: "refused"} - Runs in the native tier on the fixture composition; every test owns its disposable backend. - The test creates three documents through `CreateDocument`, which declares no read model, then runs `internal.rebuild.startGeneration` for `documentSummary` with admin access and a stated operator, and asserts one row in `generations`, generation 1, `building`, with `documentSummary`'s latest projection version and the operator as `startedBy`. -- The test runs `startGeneration` again before the first generation is switched and asserts that its error names the read model, generation 1 and its state, and that the table still holds one row. +- The test closes the tenant's gate before the first start, so the first batch parks under it, runs `startGeneration` again before the first generation is switched and asserts that its error names the read model, generation 1 and its state, and that the table still holds one row, then reopens the gate and runs `resumeGeneration`. - The test waits until the row is `verified`, asserts that `listDocumentSummaries` still fails for want of an active generation, runs `switchGeneration`, and asserts that the row is `active`, that no row is `retired`, and that `listDocumentSummaries` returns the three documents under their status. - The test runs the same sequence on a backend with one tenant and no subject and asserts that the progress row's `batchesDone` is 2, one backfill batch and one verify batch, before `verified`. diff --git a/design/specs/application/projection-contract.sdp.md b/design/specs/application/projection-contract.sdp.md index 6770c7c..e1c59ce 100644 --- a/design/specs/application/projection-contract.sdp.md +++ b/design/specs/application/projection-contract.sdp.md @@ -84,7 +84,7 @@ The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape w - validatorAggregateRowConventions: `const aggregateRowConventions = { tenantId: v.string(), generation: v.number(), key: v.string(), projectionVersion: v.number(), value: v.number(), entityCount: v.number() }` spread into every aggregate read-model `defineTable`, with the same `by_key` index, where `key` is `aggregateKeyOf(tenantId, dto)` (E-9, Law 11, D9) - tableOrderTotals: `orderTotals: defineTable({ ...aggregateRowConventions }).index("by_key", ["tenantId", "generation", "key"])` an aggregate of order totals per status, where `key` is the status, `value` the sum of the orders' totals and `entityCount` the number of orders in it, given as the shape of an aggregate table and not as a read model the first experiment builds (E-9, D9) - validatorOrderSummaryFields: `const orderSummaryFields = { orderId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), lineCount: v.number(), total: v.number(), placedAt: v.number() }` the projected fields of the order summary, which its table spreads and its query's view validator spreads (E-9, E-46, First experiment) -- tableOrderSummaries: `orderSummaries: defineTable({ ...rowConventions, ...orderSummaryFields }).index("by_key", ["tenantId", "generation", "key"]).index("by_status", ["tenantId", "generation", "status", "placedAt"])` (E-9, E-46, First experiment) +- tableOrderSummaries: `orderSummaries: defineTable({ ...rowConventions, ...orderSummaryFields }).index("by_key", ["tenantId", "generation", "key"]).index("by_status", ["tenantId", "generation", "status", "placedAt", "key"])` (E-9, E-46, First experiment) - indexByKeyUse: point read by `applyProjection` before every write and by the entity query; `key` is `keyOf(tenantId, dto)`, the application ID of the entity (E-9, D2) - indexByStatusUse: the essential list query of the first experiment, paginated, within the active generation (E-9, First experiment) - step1: for each generation in `generations`, `applyProjection` takes `projectionOf(readModel, generation.projectionVersion)` and computes `key` and the candidate row with that projection's `keyOf` and `project`; a `null` candidate means delete the row if present, answering `deleted`, and `unchanged` when there is none, in the two live modes and every role; in the backfill mode the version compare of `step3` comes first, so a present row is deleted only when its `sourceVersions` are older than the input's and answers `skipped-newer` otherwise (E-9, D8) diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index e088ed2..48dfb7b 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -46,8 +46,8 @@ This Spec carries the read-need table as rules, the properties of a projection, ### Open questions -- [non-blocking] Probe 5 showed that the built-in `.paginate` treats `maximumRowsRead` as advice once a page has an end cursor: a pinned page that grew to 25 rows came back whole with `SplitRecommended` under a cap of 20; so a read-model list is bounded in rows on its first run, and afterwards by the client's split and by Convex's own limits (Probe 5, F13, D8) -- [non-blocking] Extension E-24, as it reaches read models: a list over a parent read-model table calls the built-in `.paginate` with the options of `boundedPage`, a row cap of twice the item cap, and returns through `paginationResultValidator` with its rows mapped by `readModelView`, and a client pages every list by an explicit cursor pair with no hook; these are options taken here and the owner has not ruled (E-24, S4, Probe 5, D8) +- [non-blocking] Probe 5 showed that the built-in `.paginate` treats `maximumRowsRead` as advice once a page has an end cursor and that a cursor it issued is refused by any other query, so a subscription over the active generation threw `InvalidCursor` at a switch; a parent list therefore pages with `paginator` of `convex-helpers`, as a context list does, and a pinned page is bounded by the row cap and the byte cap on every run and answers `SplitRequired` beyond them (Probe 5, F13, D8, D9) +- [non-blocking] Extension E-24, as it reaches read models: a list over a parent read-model table calls `paginator` of `convex-helpers` with the options of `boundedPage`, a row cap of twice the item cap, with each cursor of the client's pair moved into the active generation by `pageInGeneration`, and returns through `paginationResultValidator` with its rows mapped by `readModelView`, and a client pages every list by an explicit cursor pair with no hook; these are options taken here and the owner has not ruled (E-24, S4, Probe 5, D8) - [non-blocking] Extension E-6, as it reaches queries: a parent query establishes and authorizes its actor through `authorizeQuery`, and its refusal is the command's rejection shape, `RejectionData` with the query's name in the field `commandType` and the codes `unauthenticated` or `forbidden`; the field keeps its name for a query, which the owner has not ruled on, and nothing else of the shape is new (E-6, D7, D11, Law 5) - [non-blocking] Probe 3 pending: the cost of one component call in latency and function-call quota decides whether one entity's detail stays a component query or hot reads move, which is OQ1 (Probe 3, F4, OQ1) - [non-blocking] Extension E-45: the doc says a small cross-context view is a parent query over bounded component reads and does not give the bound; the option taken here is at most 8 component reads per parent query and at most 4 read-model rows written per command, both measured by the first experiment (E-45, D8) @@ -74,6 +74,8 @@ This Spec carries the read-need table as rules, the properties of a projection, - rule: Every read-model row names its tenant, and every query on a read model takes `tenantId` as an argument and reads through an index that leads with it; an absent tenant is never a wildcard (Law 11, D11) - rule: A parent query establishes and authorizes its actor through `authorizeQuery` before its first read, then reads read-model rows by index or calls context queries with the tenant as argument; a refusal is a rejection and nothing is disclosed (D11, F11, Law 5, D7) - rule: A subscription synchronizes state; it is not durable event delivery, and no consumer treats a query result as proof of having seen every event (F7, D2) +- rule: A client's page of a read-model list names its position by the rows' order field and `key` within the tenant and never by the generation, so across a switch and a switch back the first page and every later page show the active generation's rows from the same position, a pinned page the rows up to its end cursor, and no client restarts its paging (D9, D8, E-24) +- rule: A page is read in one transaction, so its rows are one generation's; a cursor used after a switch continues from the same position in the generation then active, so at the seam no row is shown twice or skipped and a row the new generation holds differently appears as that generation holds it (D9, D8, F7) - rule: [extension] A parent query makes at most 8 component reads, and a command writes at most 4 read-model rows; a read that needs more is a read model or is outside the baseline (E-45, D8) - flow: A client subscribes to a parent query with its tenant scope and the key of the entity, list or view it needs (D8, Law 11) - flow: The query authenticates and authorizes in the parent and returns nothing before authorization passes (Law 5, D11) @@ -91,7 +93,9 @@ Nothing here schedules, and nothing here reads `ctx.auth` inside a component. Th - transactionBoundary: read-model writes happen inside the command's top-level mutation, after the context calls; reads are parent queries, reactive by subscription (D8, D1, F7) - convexSurface: one static `query` export per read need in the parent; component queries are reached through `ctx.runQuery(components.orders.queries.order.get, args)`, its siblings `list`, `history` and `rebuild`, and `components.orders.queries.operations.byOperation`, as the queries contract names them; no scheduled function, no cron, no action (D8, F11) - fnEntityDetail: `getOrder` as `spec:context.queries` pins it under `parentGet`: it calls `authorizeQuery` and then `ctx.runQuery(components.orders.queries.order.get, { tenantId, streamId: orderId })` (D8, D11, F15) -- fnListOrSummary: `export const listOrderSummaries = query({ args: { tenantId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(orderSummaryView), handler: async (ctx, { tenantId, status, paginationOpts }) => { await authorizeQuery(ctx, { name: "listOrderSummaries", tenantId, permission: "orders.read" }); const generation = await activeGeneration(ctx, "orderSummary"); if (generation === undefined) throw new Error("The read model orderSummary has no active generation"); const result = await ctx.db.query("orderSummaries").withIndex("by_status", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("status", status)).paginate(boundedPage(paginationOpts, limitReadModelList(orderSummary))); return { ...result, page: result.page.map(readModelView) }; } })` one built-in `.paginate` over the active generation's rows, in `placedAt` order within a status, where `orderSummary` is the read model, whose name is `"orderSummary"` and whose table is `orderSummaries` (D8, D9, Law 5, Law 11) +- fnListOrSummary: `export const listOrderSummaries = query({ args: { tenantId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(orderSummaryView), handler: async (ctx, { tenantId, status, paginationOpts }) => { await authorizeQuery(ctx, { name: "listOrderSummaries", tenantId, permission: "orders.read" }); const generation = await activeGeneration(ctx, "orderSummary"); if (generation === undefined) throw new Error("The read model orderSummary has no active generation"); const result = await paginator(ctx.db, schema).query("orderSummaries").withIndex("by_status", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("status", status)).paginate(boundedPage(pageInGeneration(paginationOpts, generation), limitReadModelList(orderSummary))); return { ...result, page: result.page.map(readModelView) }; } })` one `paginate` of `paginator` from `convex-helpers/server/pagination` with the composition's `schema`, over the active generation's rows in `placedAt` then `key` order within a status, where `orderSummary` is the read model, whose name is `"orderSummary"` and whose table is `orderSummaries` (D8, D9, Law 5, Law 11, E-24) +- fnPageInGeneration: `pageInGeneration(opts: PaginationOptions, generation: number): PaginationOptions` returns `opts` with its `cursor` and its `endCursor`, when set, each moved into `generation`: a cursor of `paginator` is the JSON of a row's index key, so the function replaces the key's second element, the generation that wrote the row, with `generation` and drops the last two, `_creationTime` and `_id`, which belong to that generation's document; a cursor that is not such a key, `null` or `undefined` is returned as it is, and the read-model library exports the function beside `readModelView` (D9, E-24, Probe 5) +- indexReadModelList: the index a read-model list reads has `tenantId`, `generation` and the list's equality fields first, then its order field, then `key`, as `by_status` of `tableOrderSummaries` in `spec:application.projection-contract`, so a position within a tenant names at most one row of each generation and a cursor moved into another generation lands between the same rows there (D9, E-9, Law 11) - fnReadModelView: `readModelView(row: Row): Omit` what a read-model query returns for a row: the tenant, the key, the projection version, the source versions and the projected fields; the built-in page's rows carry `_id` and `_creationTime`, which an object validator of the row's fields refuses, and no document ID or generation number leaves the parent (D8, D2, E-9) - validatorReadModelView: `orderSummaryView = v.object({ tenantId: v.string(), key: v.string(), projectionVersion: v.number(), sourceVersions: v.array(streamVersionValidator), ...orderSummaryFields })` where `orderSummaryFields` are the projected fields `spec:application.projection-contract` declares for `tableOrderSummaries` (E-9, D8) - noActiveGeneration: a query over a read model that has no active generation throws a plain `Error` that names the read model, a technical failure, and never answers an empty page (D9, Law 6) @@ -100,11 +104,11 @@ Nothing here schedules, and nothing here reads `ctx.auth` inside a component. Th - readModelRowShape: every read-model row carries `tenantId`, `generation`, `key`, `projectionVersion` and the projected fields, a per-entity or cross-context row carries `sourceVersions`, and an aggregate row carries `value` and `entityCount` with the versions on its markers, as the projection contract pins (E-9, D8, D9) - indexReadModelByKey: `.index("by_key", ["tenantId", "generation", "key"])` on every read-model table (E-9, Law 11) - indexReadModelByKeyUse: point read of one row by the parent query and by `applyProjection` before an update; the generation comes from the registry's active row, which is one per read model and the same for every tenant, so a query narrows by its own `tenantId` and never by a tenant's generation (D8, D9, E-8, Law 11) -- pagination: every list returns through `paginationResultValidator(item)` of `convex/server` and passes its one `paginate` call the options of `boundedPage`, as `spec:context.queries` pins; a list over a parent read-model table calls the built-in `.paginate`, whose result always carries `pageStatus` and `splitCursor`, and maps the page's rows through `readModelView`; a list served by a context relays the context's page; a client pages either by an explicit cursor pair; a function runs at most one built-in `.paginate`, because Convex refuses a second with "Convex only supports a single paginated query in each function" (D8, S4, Probe 5, F15, E-24) +- pagination: every list returns through `paginationResultValidator(item)` of `convex/server` and passes its one `paginate` call the options of `boundedPage`, as `spec:context.queries` pins; a list over a parent read-model table calls `paginator` of `convex-helpers` with the composition's schema, as a context list does, over the index of `indexReadModelList`, with the client's cursors moved into the active generation by `pageInGeneration`, relays `pageStatus` and `splitCursor` as the helper returns them, and maps the page's rows through `readModelView`; a list served by a context relays the context's page; a client pages either by an explicit cursor pair; a function runs at most one built-in `.paginate`, because Convex refuses a second with "Convex only supports a single paginated query in each function" (D8, S4, Probe 5, F15, E-24) - limitReadModelList: `limitReadModelList(readModel) = { items: limitListPage(readModel.rowBudgetBytes), bytes: limitListBytes }`, a function of the read model, so 100 items at the default 16 KiB and 64 at the 64 KiB cap; the built-in honours the item cap on a first run and answers `SplitRecommended` for a pinned page that has outgrown the row cap (F13, E-9, Probe 5) - limitComponentReadsPerQuery: `[extension]` 8 component reads per parent query (E-45, D8) - limitReadModelWritesPerCommand: `[extension]` 4 read-model rows per command, counted as the rows step 9 inserts, replaces or deletes, a row in each generation once, and checked after each `streams` entry is applied; above it step 9 throws a plain error and nothing of the command commits (E-45, D8, Sc L2-3) -- readModelVisibility: a subscription re-runs when a row it read changes, including on generation switch, because the query reads the registry row too (D8, F7, D9) +- readModelVisibility: a subscription re-runs when a row it read changes, including on generation switch, because the query reads the registry row too; after the switch and after a switch back each page shows the rows of the generation then active from the position its cursors name, so a client sees the same orders and never `InvalidCursor` (D8, F7, D9) - noProjectionJob: no scheduled function, cron, action or component maintains a read model on the core path; the only batch writer of read models is the rebuild (D8, D9) ## Example space @@ -132,3 +136,4 @@ And the number of pages that carry pageStatus SplitRequired is {splitPages:numbe - A reviewer confirms that no scheduled function, cron, action or component maintains a read model on the core path. - A reviewer confirms that every read-model index leads with `tenantId`, that every query over a read model takes `tenantId` in its `args` and calls `authorizeQuery` before its first read, and that every list returns through `paginationResultValidator` with rows mapped by `readModelView`. - A reviewer confirms that F15 is cited wherever a parent query calls a component query. +- A reviewer confirms that every read-model list pages with `paginator`, that the index it reads ends with `key` after its order field, and that it moves both cursors of the client's pair into the active generation. diff --git a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md index 45b44dc..86a1bed 100644 --- a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md +++ b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md @@ -33,6 +33,6 @@ And writes {writes: "never stopped"} - The test, in a file whose own timeout is 300 s, installs the order summary's generation 1 through its first rebuild, seeds 350 orders across two tenants, 250 in the first in tenant-ID order and 100 in the second, starts generation 2 with `projectionVersion` 1 stated, a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that `batchesDone` then stops advancing below 350 and that the cursor names a tenant; while a chain is expected to run, a progress row whose `batchesDone` has not advanced for 30 s fails the test with the generation row and the progress row in the message. - While the chain is interrupted the test sends `CancelOrder` to one of the second tenant's 100 seeded orders and to an order of the first tenant whose row generation 2 already holds, read from the table before the command; it asserts at once that the first tenant's order has its row in generation 2 `cancelled`, and, once generation 2 is `verified`, that the second tenant's order has its row in generation 2 `cancelled`, whichever pass wrote it. - After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that, once the generation is `verified`, every order the Orders context's `list` enumerates in either tenant, read with admin access and `includeDeleted`, has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, that `backfillBatch` run with admin access under the fence the interrupt replaced leaves the progress row unchanged, that the function log shows no action and no completion during the batches other than a batch, an operator entry or a live command, and that the journals, read by operation, hold the event and receipt counts the live commands produced. -- The test asserts that no command of the run ends in a failure, that no completion record's `occInfo` names `generationProgress`, and that the completion records whose `occInfo` names `generations` number at most the operator entries and pass ends of the run times the callers the test keeps in flight plus one for the chain, and records those reruns with their `occInfo` as a measurement; it reads generation 2's row at the interrupt, after the resume and at `verified`, and asserts that the interrupt and the resume changed only `fence`, `changedAt` and `changedBy` and that from the resume to `verified` only `state` changed. +- The test asserts that no command of the run ends in a failure, that no completion record of a `PlaceOrder`, a `CancelOrder` or a parent query names `generationProgress` in its `occInfo`, and that the completion records whose `occInfo` names `generations` number at most the operator entries and pass ends of the run times the callers the test keeps in flight plus one for the chain, and records those reruns with their `occInfo` as a measurement, beside them the count of the reruns of operator entries and batches whose `occInfo` names `generationProgress`, each with its `occInfo`; it reads generation 2's row at the interrupt, after the resume and at `verified`, and asserts that the interrupt and the resume changed only `fence`, `changedAt` and `changedBy` and that from the resume to `verified` only `state` changed. - The test switches, asserts from the function log that the list query ran again for each tenant's subscription after the switch and that each subscription's value holds the same rows, and that generation 1 is `retired`, then runs one more `PlaceOrder` and cancels one earlier order so that its summary changes, rolls back by naming generation 1, asserts that the reopened generation reaches `verified` after one verify pass with a row for the new order and the cancelled order's row `cancelled` before it becomes active, switches again, and asserts that generation 1 is active, generation 2 is `retired` and the list query ran again for each subscription. - A `convex-test` test of the same workflow runs on the fixture composition with `documentTitle` and calls each batch directly, so it interrupts after exactly 3 batches; in one run generation 1 is installed at version 1 stated and generation 2 built at version 2 under live `CreateTwiceSummarizedDocument` writes, and it asserts that each generation's rows carry their own generation's `projectionVersion` and that a document whose `title` is the depot's `deletedTitle` has a row at version 1 and none at version 2; in a second run both generations are at version 2 and, after the switch, it deletes a document by `AmendDocument` with the depot's `deletedTitle`, rolls back by naming generation 1 and asserts that the reopened generation holds no row for it after its verify pass; and it asserts every refusal of the operator entries by its message. diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index c24d40f..40e8ea2 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -85,7 +85,7 @@ The registry, the progress row, the per-entity marker and the gate document are - rule: A backfill batch racing a live command is ordered by optimistic concurrency; the batch's read-then-write of a row is one mutation, so the batch never overwrites a row a command wrote after the batch read its source (D9, F1) - rule: [extension] Every batch, whether its source is a context's enumeration or a parent table, is driven by a self-scheduled internal mutation under one batch contract, and the rebuild uses `@convex-dev/migrations` for nothing (E-40, F17, F8) - rule: [extension] A projection version change is a new generation, never an in-place rewrite of the active one (E-8, D8) -- rule: [extension] The batch bound is a field of the progress row, a whole number of at least 1 and at most `rebuildBatchCeiling`, and never more than the context's `list` page admits at the stream type's document budget; `startGeneration` sets it and nothing changes it after, so every pass and every resumed pass runs at the size on the row, and a purge batch deletes at most that many rows, capped by `limitPurgeBatch` (E-40, E-24, E-9, F1, F13) +- rule: [extension] The batch bound is a field of the progress row, the read model's by `batchSizeFor`, a whole number of at least 1 and at most `rebuildBatchCeiling(readModel)`; the context's `list` page bounds the bytes a batch reads, so a batch may take fewer entities than its size, and does not change the size; `startGeneration` sets it and nothing changes it after, so every pass and every resumed pass runs at the size on the row, and a purge batch deletes at most that many rows, capped by `limitPurgeBatch(readModel)` (E-40, E-24, E-9, F1, F13) - rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them returns without writing (E-40, E-8, F8) - An operator calls `startGeneration` for a read model; it inserts the generation row in state `building` with the next generation number, the projection version and fence 1, inserts the progress row in the pass `backfill` with the batch size and a `null` cursor, and schedules the first backfill batch, all in one mutation; a read model with no generation gets generation 1 this way (D9, F8, E-8) - [extension] On a deployment whose grants predate the tenant list an operator calls `fillTenants` before the first `startGeneration`; it inserts the `tenantFill` row, or reopens it, in the pass `fill` with a new fence and schedules the first fill batch; each fill batch reads the gate, takes up to `limitTenantFillBatch` further tenants from the grants after the row's cursor, inserts the tenant row of each that has none, saves the last tenant ID as the cursor and schedules the next fill batch, and the batch that finds no further tenant sets the pass to `idle` and schedules nothing (E-49, E-48, E-40, F8) @@ -104,7 +104,7 @@ The registry, the progress row, the per-entity marker and the gate document are ## Design -Each batch is one scheduled internal mutation in the parent. The rebuild schedules from a mutation, so a batch is scheduled only if its predecessor committed. The rebuild is an operator's work on parent tables; it is not deferred business work and creates no obligation. The library exports one plain helper per entry below, of the same name, taking `(ctx, config: RebuildConfig, args)` and returning the entry's value; a composition's `rebuild` module registers each entry over its helper with its own `internalMutation` and `internalQuery`, so the entries are reached as `internal.rebuild.startGeneration` and so on, and no public function starts or changes a generation. Every refusal is a plain `Error` with the message given; in a message, `name` is the read model's name, `n` the generation number and `state` the generation's state. Beside the helpers the library exports `batchSizeFor`, `rebuildBatchCeiling`, `limitPurgeBatch`, `resumeChain`, the constants `limitTenantFillBatch` and `limitGenerationsListed`, and the types `RebuildConfig`, `RebuildTarget`, `BatchRef` and `FillBatchRef`. Every patch of a progress row or of the `tenantFill` row sets `updatedAt` to the writing mutation's `Date.now()`; a field written as absent, `lastError`, `retiredAt` or `retireAfter`, is removed from the document, which a patch with `undefined` does; a batch never writes `changedAt` or `changedBy`, which only an operator entry sets, and when it ends a pass it patches the generation row's `state` and nothing else on that row. +Each batch is one scheduled internal mutation in the parent. The rebuild schedules from a mutation, so a batch is scheduled only if its predecessor committed. The rebuild is an operator's work on parent tables; it is not deferred business work and creates no obligation. The library exports one plain helper per entry below, of the same name, taking `(ctx, config: RebuildConfig, args)` and returning the entry's value; a composition's `rebuild` module registers each entry over its helper with its own `internalMutation` and `internalQuery`, so the entries are reached as `internal.rebuild.startGeneration` and so on, and no public function starts or changes a generation. Every refusal is a plain `Error` with the message given; in a message, `name` is the read model's name, `n` the generation number and `state` the generation's state. Beside the helpers the library exports `batchSizeFor`, `rebuildBatchCeiling`, `limitPurgeBatch`, `resumeChain`, the constants `limitTenantFillBatch` and `limitGenerationsListed`, and the types `RebuildConfig`, `RebuildTarget`, `BatchRef` and `FillBatchRef`. Every patch of a progress row or of the `tenantFill` row sets `updatedAt` to the writing mutation's `Date.now()`; a field written as absent, `lastError`, `retiredAt` or `retireAfter`, is removed from the document, which a patch with `undefined` does; a batch never writes `changedAt` or `changedBy`, which only an operator entry sets, and when it ends a pass it patches the generation row's `state`, and at the end of a purge also removes `retiredAt` and `retireAfter`, and nothing else on that row. A batch, and an operator entry that reads the progress row, over a generation that has no progress row throws `"No progress for generation " + generationId`. The enumeration is the context's `list` query with `includeDeleted`, the one read of context data per batch: one call returns a page of up to `batchSize` DTOs and a continuation cursor, built inside the component by `paginator` from `convex-helpers` over the `by_identity` index that includes deleted subjects, and capped by the query at its byte-derived page bound. A batch reads `page`, `isDone` and `continueCursor` and ignores any split status, because the rows up to `continueCursor` are complete. A DTO carries the stream version the projection compares, as the queries contract's `toDto(state, meta)` maps it onto `version`. Deleted subjects reach the projection as DTOs the projection maps to `null`, so their rows are deleted rather than left behind. @@ -125,7 +125,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - step5: for each DTO of the page it calls `applyProjection(ctx, target.readModel, { tenantId, dto, versions: [dto.version], mode: "backfill", generations: [thisGeneration] })` for this generation alone, where `thisGeneration` is `{ role, generation: n, projectionVersion }` with the role `typeWritableGeneration` of `spec:application.generation-registry` gives the generation's state, adding one to `rowsWritten` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` for `skipped-newer` or `unchanged` (D9, E-9) - step6: it checkpoints: when the page is not done the cursor becomes `{ tenantId, pageCursor: continueCursor }`; when it is done the cursor becomes `{ tenantId: next, pageCursor: null }` for `next = nextTenant(ctx, tenantId)`, and the pass is done when `next` is `null`; a pass that is not done patches the progress row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.backfillBatch` with the same fence through `ctx.scheduler.runAfter(0, ...)`; a pass that is done patches the generation row to `state: "verifying"` and nothing else on it, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules `refs.verifyBatch` with the same fence (D9, F8, E-40, E-49) - verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `pass` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) -- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(Math.min(batchSize, limitPurgeBatch))` with `batchSize` the progress row's, deletes each row and adds their number to `rowsPurged`; when it deleted that many rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`, a purge cursor being `{ tenantId, pageCursor: null }`; a pass that is not done checkpoints, patching the progress row with the cursor, `rowsPurged`, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) +- purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(Math.min(batchSize, limitPurgeBatch(readModel)))` with `batchSize` the progress row's and `readModel` the generation's read model, whose cap `limitPurgeBatch` declares, deletes each row and adds their number to `rowsPurged`; when it deleted that many rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`, a purge cursor being `{ tenantId, pageCursor: null }`; a pass that is not done checkpoints, patching the progress row with the cursor, `rowsPurged`, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) - purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) - fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator` and `updatedAt`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) - fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to no `lastError` and `updatedAt` and leaving its `batchSize`, cursor and counts as they are, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) @@ -133,8 +133,8 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) - fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `pass` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, the in-flight error of `startGeneration`, naming that generation's number and state, when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, no `lastError` and `updatedAt`, with `batchSize` and the other counts kept, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) - fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and `"Generation " + n + " of " + name + " is " + state + "; abortGeneration needs building, verifying or verified"` unless the state is one of the three; otherwise patching the row to `state: "aborted"`, fence plus 1, `changedAt` and `changedBy`, patching the progress row to `pass: "idle"`, `lastError: "aborted: " + reason` and `updatedAt`, with the cursor and the counts kept, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8, E-48) -- fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged; resumeGeneration continues it"` when `pass` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `pass: "purge"`, `cursor: null` and no `lastError`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) -- fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(v.object({ generation: generationDoc, progress: progressDoc })), handler })` the newest `limitGenerationsListed` generations of the read model by `by_read_model` in descending order, each with its progress row, where `generationDoc` and `progressDoc` are the two tables' document validators with `_id` and `_creationTime`; an operator entry that only reads, so it takes no operator (E-8, E-40, E-48, D19) +- fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged; resumeGeneration continues it"` when `pass` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `pass: "purge"`, `cursor: null`, no `lastError` and `updatedAt`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) +- fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(v.object({ generation: generationDoc, progress: progressDoc })), handler })` the newest `limitGenerationsListed` generations of the read model by `by_read_model` in descending order, each with its progress row, where `generationDoc` and `progressDoc` are the two tables' document validators with `_id` and `_creationTime`, and `[]` for a read model the composition does not declare; an operator entry that only reads, so it takes no operator (E-8, E-40, E-48, D19) - tableTenantFill: `tenantFill: defineTable({ pass: v.union(v.literal("fill"), v.literal("idle")), fence: v.number(), cursor: v.union(v.string(), v.null()), batchesDone: v.number(), tenantsRead: v.number(), tenantsInserted: v.number(), lastError: v.optional(v.string()), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), updatedAt: v.number() })` in `readModelTables`, at most one row per deployment, the progress row of the tenant fill and nothing else's; operational data of the deployment with no tenant dimension and no index, read with `ctx.db.query("tenantFill").first()`; `cursor` is the last tenant ID the walk passed, `null` before its first, `tenantsRead` counts the tenants the walk found and `tenantsInserted` the rows it inserted (E-49, E-40, E-48) - fnFillTenants: `export const fillTenants = internalMutation({ args: { operator: v.string() }, returns: v.null(), handler })` in order: `assertOperator`; the restore door, throwing `"fillTenants is refused while the restore door is closed"` when `process.env.MAINTENANCE_MODE` is exactly `restore`, because the fill writes the tenant list that a restore replaces, and no other read of the gate, because the fill writes no tenant's data and so no closed tenant scope stops it; for each target of `config.targets` the in-flight check by `by_read_model_state`, throwing `"Read model " + name + " has generation " + n + " in " + state + "; fillTenants needs no generation in flight"`; then, with `now` the mutation's `Date.now()`, when no `tenantFill` row exists it inserts one with `pass: "fill"`, `fence: 1`, `cursor: null`, every count 0, `startedAt`, `changedAt` and `updatedAt` at `now` and `startedBy` and `changedBy` the operator, and when one exists it patches it to `pass: "fill"`, fence plus 1, the cursor and the counts kept when the pass was `fill` and `null` and 0 when it was `idle`, no `lastError`, `changedAt`, `changedBy` and `updatedAt`; then it schedules `refs.fillTenantsBatch` with the row's fence through `ctx.scheduler.runAfter(0, ...)`; so a call continues a fill whose chain a restore, an error or a closed gate ended, and a call after a finished fill walks the grants again and inserts only what is missing (E-49, E-48, E-40, F8) - fnFillTenantsBatch: `export const fillTenantsBatch = internalMutation({ args: { fence: v.number() }, returns: v.null(), handler })` (E-49, E-40) diff --git a/tests/native/generation-registry.first-rebuild-installs-read-model.test.ts b/tests/native/generation-registry.first-rebuild-installs-read-model.test.ts index c5b4b10..72dfca2 100644 --- a/tests/native/generation-registry.first-rebuild-installs-read-model.test.ts +++ b/tests/native/generation-registry.first-rebuild-installs-read-model.test.ts @@ -86,7 +86,7 @@ export async function plainNativeMessage(call: Promise) { } async function rebuild(world: World) { const backend = required(world.backend, "the backend"); - // rebuild.sdp.md:123: park the batch so the building row can be observed without a scheduler race. + // generation-registry.first-rebuild-installs-read-model.sdp.md:30; rebuild.sdp.md:123: the closed tenant gate parks the first batch. await backend.admin.run(getFunctionName(internal.gate.closeGate), { scopeKey: `tenant:${tenantId}`, reason: "observe the first generation", @@ -129,7 +129,7 @@ async function rebuild(world: World) { expect(await backend.admin.readTable("generations")).toEqual( entries.map(({ generation }) => generation), ); - // Let the scheduled batch park before reopening the gate. + // generation-registry.first-rebuild-installs-read-model.sdp.md:30: the first batch parks before the gate reopens and the chain resumes. await waitForGeneration( backend, "documentSummary", From d85c0fe57f77c6d1e9e65f624dc65a588a4f39da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:50:03 +0200 Subject: [PATCH 090/155] build: 3b fold-in a read-model list pages with paginator and moves the client's cursors into the active generation pageInGeneration replaces the generation of each cursor's index key with the active one and drops the document's _creationTime and _id; by_status of orderSummaries ends with key. A convex-test case covers a later page, a pinned page, the switch back and a tie on placedAt. Co-Authored-By: Claude Opus 5.5 --- example/convex/readModels.ts | 12 +- example/convex/schema.ts | 8 +- fixture/convex/readModels.ts | 10 +- src/read-model/index.ts | 6 +- src/read-model/queries.ts | 26 ++++ .../read-model-page-in-generation.test.ts | 116 ++++++++++++++++++ 6 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 tests/simulator/read-model-page-in-generation.test.ts diff --git a/example/convex/readModels.ts b/example/convex/readModels.ts index 63c6f2c..3f04a73 100644 --- a/example/convex/readModels.ts +++ b/example/convex/readModels.ts @@ -5,6 +5,7 @@ import { paginationResultValidator, } from "convex/server"; import { v } from "convex/values"; +import { paginator } from "convex-helpers/server/pagination"; import { authorizeQuery } from "../../src/command/index.js"; import { boundedPage, @@ -13,9 +14,11 @@ import { import { activeGeneration, limitReadModelList, + pageInGeneration, readModelView, } from "../../src/read-model/index.js"; import { query } from "./_generated/server.js"; +import schema from "./schema.js"; import { orderSummary, orderSummaryFields, @@ -48,7 +51,7 @@ export const listOrderSummaries = query({ throw new Error( `The read model ${orderSummary.name} has no active generation`, ); - const result = await ctx.db + const result = await paginator(ctx.db, schema) .query("orderSummaries") .withIndex("by_status", (q) => q @@ -56,7 +59,12 @@ export const listOrderSummaries = query({ .eq("generation", generation) .eq("status", status), ) - .paginate(boundedPage(paginationOpts, limitReadModelList(orderSummary))); + .paginate( + boundedPage( + pageInGeneration(paginationOpts, generation), + limitReadModelList(orderSummary), + ), + ); return { ...result, page: result.page.map(readModelView) }; }, }); diff --git a/example/convex/schema.ts b/example/convex/schema.ts index 2bb81fd..d483780 100644 --- a/example/convex/schema.ts +++ b/example/convex/schema.ts @@ -14,5 +14,11 @@ export default defineSchema({ ...readModelTables, orderSummaries: defineTable({ ...rowConventions, ...orderSummaryFields }) .index("by_key", ["tenantId", "generation", "key"]) - .index("by_status", ["tenantId", "generation", "status", "placedAt"]), + .index("by_status", [ + "tenantId", + "generation", + "status", + "placedAt", + "key", + ]), }); diff --git a/fixture/convex/readModels.ts b/fixture/convex/readModels.ts index 54097a5..44fb185 100644 --- a/fixture/convex/readModels.ts +++ b/fixture/convex/readModels.ts @@ -5,15 +5,18 @@ import { paginationResultValidator, } from "convex/server"; import { v } from "convex/values"; +import { paginator } from "convex-helpers/server/pagination"; import { authorizeQuery } from "../../src/command/index.js"; import { boundedPage } from "../../src/context/index.js"; import { streamVersionValidator } from "../../src/context/index.js"; import { activeGeneration, limitReadModelList, + pageInGeneration, readModelView, } from "../../src/read-model/index.js"; import { query } from "./_generated/server.js"; +import schema from "./schema.js"; import { documentSummary, documentSummaryFields, @@ -46,7 +49,7 @@ export const listDocumentSummaries = query({ throw new Error( `The read model ${documentSummary.name} has no active generation`, ); - const result = await ctx.db + const result = await paginator(ctx.db, schema) .query("documentSummaries") .withIndex("by_status", (q) => q @@ -55,7 +58,10 @@ export const listDocumentSummaries = query({ .eq("status", status), ) .paginate( - boundedPage(paginationOpts, limitReadModelList(documentSummary)), + boundedPage( + pageInGeneration(paginationOpts, generation), + limitReadModelList(documentSummary), + ), ); return { ...result, page: result.page.map(readModelView) }; }, diff --git a/src/read-model/index.ts b/src/read-model/index.ts index cbfee0e..2f008d8 100644 --- a/src/read-model/index.ts +++ b/src/read-model/index.ts @@ -15,7 +15,11 @@ export type { ReadModel, WritableGeneration, } from "./projection.js"; -export { limitReadModelList, readModelView } from "./queries.js"; +export { + limitReadModelList, + pageInGeneration, + readModelView, +} from "./queries.js"; export { batchCursorValidator, generationStateValidator, diff --git a/src/read-model/queries.ts b/src/read-model/queries.ts index 560dbae..422d025 100644 --- a/src/read-model/queries.ts +++ b/src/read-model/queries.ts @@ -5,6 +5,7 @@ import { limitListPage, type PageLimit, } from "../context/queries.js"; +import type { PaginationOptions } from "convex/server"; import type { AnyReadModel } from "./projection.js"; // A read-model row as a query returns it: the document ID, its creation time and the generation stay // in the parent. @@ -20,3 +21,28 @@ export const limitReadModelList = (readModel: AnyReadModel): PageLimit => ({ items: limitListPage(readModel.rowBudgetBytes), bytes: limitListBytes, }); +// A client's page options with each cursor moved into the generation now active. A cursor of +// paginator is the JSON of a row's index key, whose second element is the generation that wrote the +// row and whose last two are that document's _creationTime and _id; the moved cursor names the same +// position by the order field and key alone. +export function pageInGeneration( + opts: PaginationOptions, + generation: number, +): PaginationOptions { + const moved = { ...opts, cursor: moveCursor(opts.cursor, generation) }; + if (opts.endCursor !== undefined && opts.endCursor !== null) + moved.endCursor = moveCursor(opts.endCursor, generation); + return moved; +} +function moveCursor(cursor: string | null, generation: number) { + if (cursor === null) return cursor; + let key: unknown; + try { + key = JSON.parse(cursor); + } catch { + return cursor; + } + if (!Array.isArray(key) || key.length < 4 || typeof key[1] !== "number") + return cursor; + return JSON.stringify([key[0], generation, ...key.slice(2, -2)]); +} diff --git a/tests/simulator/read-model-page-in-generation.test.ts b/tests/simulator/read-model-page-in-generation.test.ts new file mode 100644 index 0000000..4a7621d --- /dev/null +++ b/tests/simulator/read-model-page-in-generation.test.ts @@ -0,0 +1,116 @@ +// read-models.sdp.md fnPageInGeneration, indexReadModelList, pagination and the two rules on a +// client's page across a switch; projection-contract.sdp.md tableOrderSummaries. +import type { PaginationOptions } from "convex/server"; +import { paginator } from "convex-helpers/server/pagination"; +import { expect, test } from "vitest"; +import schema from "../../example/convex/schema.js"; +import { pageInGeneration } from "../../src/read-model/index.js"; +import { productionTest } from "./production.js"; + +const tenantId = "tenant-a"; +// b and c share placedAt, so a page edge between them is a tie on the order field. +const orders = [ + ["a", 1], + ["b", 2], + ["c", 2], + ["d", 3], + ["e", 4], +] as const; +type Page = { rows: string[]; isDone: boolean; continueCursor: string }; + +async function seeded() { + const t = productionTest(); + await t.run(async (ctx) => { + for (const generation of [1, 2]) + for (const [key, placedAt] of orders) + await ctx.db.insert("orderSummaries", { + tenantId, + generation, + key, + projectionVersion: 1, + sourceVersions: [], + orderId: key, + status: "placed", + lineCount: 1, + total: 1, + placedAt, + }); + }); + return t; +} +// One page of the list as listOrderSummaries reads it, in the generation then active. +function page( + t: Awaited>, + generation: number, + opts: PaginationOptions, +): Promise { + return t.run(async (ctx) => { + const result = await paginator(ctx.db, schema) + .query("orderSummaries") + .withIndex("by_status", (q) => + q + .eq("tenantId", tenantId) + .eq("generation", generation) + .eq("status", "placed"), + ) + .paginate(pageInGeneration(opts, generation)); + return { + rows: result.page.map((row) => `${row.key}@${row.generation}`), + isDone: result.isDone, + continueCursor: result.continueCursor, + }; + }); +} + +test("convex-test: a later page's cursor continues from the same position in the generation switched to", async () => { + const t = await seeded(); + const first = await page(t, 1, { cursor: null, numItems: 1 }); + expect(first.rows).toEqual(["a@1"]); + expect( + (await page(t, 2, { cursor: first.continueCursor, numItems: 2 })).rows, + ).toEqual(["b@2", "c@2"]); +}); + +test("convex-test: a pinned page shows the switched-to generation's rows up to its end cursor", async () => { + const t = await seeded(); + const first = await page(t, 1, { cursor: null, numItems: 2 }); + expect(first.rows).toEqual(["a@1", "b@1"]); + const pinned = await page(t, 2, { + cursor: null, + endCursor: first.continueCursor, + numItems: 2, + }); + expect(pinned.rows).toEqual(["a@2", "b@2"]); + expect(pinned.isDone).toBe(false); +}); + +test("convex-test: after a switch back a cursor of generation 2 continues from the same position in generation 1", async () => { + const t = await seeded(); + const first = await page(t, 2, { cursor: null, numItems: 2 }); + expect(first.rows).toEqual(["a@2", "b@2"]); + expect( + (await page(t, 1, { cursor: first.continueCursor, numItems: 2 })).rows, + ).toEqual(["c@1", "d@1"]); +}); + +test("convex-test: a page edge on an order-field tie neither repeats nor skips a row across a switch", async () => { + const t = await seeded(); + const first = await page(t, 1, { cursor: null, numItems: 2 }); + expect(first.rows).toEqual(["a@1", "b@1"]); + const next = await page(t, 2, { + cursor: first.continueCursor, + numItems: 2, + }); + expect(next.rows).toEqual(["c@2", "d@2"]); +}); + +test("convex-test: pageInGeneration returns a cursor that is not an index key as it is", () => { + for (const cursor of [null, "[]", "not json", '{"a":1}', '["a","b","c","d"]']) + expect(pageInGeneration({ cursor, numItems: 1 }, 2)).toStrictEqual({ + cursor, + numItems: 1, + }); + expect( + pageInGeneration({ cursor: null, endCursor: null, numItems: 1 }, 2), + ).toStrictEqual({ cursor: null, endCursor: null, numItems: 1 }); +}); From 66407845385c2435620fb5248c97ec3329af404a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:50:03 +0200 Subject: [PATCH 091/155] build: 3b fold-in a batch over a generation with no progress row throws, and the two proof tests a lawful build passes The operator-first test guards ctx alone and passes a lawful config; the pure write fake answers .first() and .unique() alike. Co-Authored-By: Claude Opus 5.5 --- src/read-model/rebuild.ts | 3 +- tests/pure/rebuild-support.ts | 18 ++++----- tests/simulator/rebuild.test.ts | 65 ++++++++++++++++++++++++++++++--- 3 files changed, 71 insertions(+), 15 deletions(-) diff --git a/src/read-model/rebuild.ts b/src/read-model/rebuild.ts index 7a028d3..3a8a241 100644 --- a/src/read-model/rebuild.ts +++ b/src/read-model/rebuild.ts @@ -466,6 +466,8 @@ async function generationBatch( .query("generationProgress") .withIndex("by_generation", (q) => q.eq("generationId", args.generationId)) .unique(); + if (progress === null) + throw new Error(`No progress for generation ${args.generationId}`); const stateMatches = pass === "backfill" ? generation.state === "building" @@ -474,7 +476,6 @@ async function generationBatch( : generation.state === "retired" || generation.state === "aborted"; if ( !stateMatches || - progress === null || progress.pass !== pass || generation.fence !== args.fence ) diff --git a/tests/pure/rebuild-support.ts b/tests/pure/rebuild-support.ts index ae8be53..2799028 100644 --- a/tests/pure/rebuild-support.ts +++ b/tests/pure/rebuild-support.ts @@ -136,15 +136,15 @@ export function memoryWriter(initial: ReturnType) { }, }; select(q); - return { - unique: async () => - row && - equalities.every( - ([key, value]) => row![key as keyof Stored] === value, - ) - ? structuredClone(row) - : null, - }; + // The Spec pins no read method, so .first() and .unique() answer alike. + const one = async () => + row && + equalities.every( + ([key, value]) => row![key as keyof Stored] === value, + ) + ? structuredClone(row) + : null; + return { first: one, unique: one }; }, }; }, diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts index 957ff11..3a70409 100644 --- a/tests/simulator/rebuild.test.ts +++ b/tests/simulator/rebuild.test.ts @@ -3,6 +3,7 @@ import { components, internal } from "../../fixture/convex/_generated/api.js"; import { internal as exampleInternal } from "../../example/convex/_generated/api.js"; import { deletedTitle } from "../../fixture/convex/depot/streams.js"; import { documentTitle } from "../../fixture/convex/documentTitles.js"; +import { documentSource } from "../../fixture/convex/summarizedTwice.js"; import { applyProjection } from "../../src/read-model/projection.js"; import { candidateRow, @@ -308,6 +309,48 @@ test("convex-test: every generation entry refuses an unknown ID and every batch expect(await snapshot(t)).toStrictEqual(before); }); +// rebuild.sdp.md Design:107, the generation with no progress row. +test("convex-test: every entry that reads the progress row and every batch over a generation with none throws", async () => { + const t = rebuildApp(); + const id = await start(t); + const { generation, progress } = await read(t, id); + await t.run(async (ctx) => { + await ctx.db.delete(progress._id); + }); + const message = `No progress for generation ${id}`; + for (const ref of [ + internal.rebuild.interruptGeneration, + internal.rebuild.resumeGeneration, + internal.rebuild.rollbackGeneration, + internal.rebuild.purgeGeneration, + ]) + await refused( + t, + () => t.mutation(ref, { generationId: id, operator }), + message, + ); + await refused( + t, + () => + t.mutation(internal.rebuild.abortGeneration, { + generationId: id, + operator, + reason: "repair", + }), + message, + ); + for (const ref of [ + internal.rebuild.backfillBatch, + internal.rebuild.verifyBatch, + internal.rebuild.purgeBatch, + ]) + await refused( + t, + () => t.mutation(ref, { generationId: id, fence: generation.fence }), + message, + ); +}); + // actor-and-scope.sdp.md fnAssertOperator:91; rebuild.sdp.md fnStartGeneration:115 through fnFillTenants:139. test.each(["", " \n\t ", " ".repeat(513), "x".repeat(513), "é".repeat(257)])( "convex-test: every operator entry refuses invalid operator %j before a lookup", @@ -353,7 +396,7 @@ test.each(["", " \n\t ", " ".repeat(513), "x".repeat(513), "é".repeat(257)])( ); // actor-and-scope.sdp.md fnAssertOperator:91; rebuild.sdp.md convexSurface:112. -test("convex-test: every plain entry checks the operator before touching ctx or config", async () => { +test("convex-test: every plain entry checks the operator before touching ctx", async () => { const t = rebuildApp(); const id = await start(t); const helpers = await import("../../src/read-model/rebuild.js"); @@ -362,10 +405,22 @@ test("convex-test: every plain entry checks the operator before touching ctx or throw new Error("Read before assertOperator"); }); const guarded = new Proxy(ctx, { get: touched }); - const config = new Proxy( - {} as import("../../src/read-model/rebuild.js").RebuildConfig, - { get: touched }, - ); + // Reading config, a plain object, before assertOperator is lawful; only ctx is guarded. + const config: import("../../src/read-model/rebuild.js").RebuildConfig = { + targets: [ + { + readModel: documentTitle, + source: documentSource, + list: components.depot.queries.document.list, + }, + ], + refs: { + backfillBatch: internal.rebuild.backfillBatch, + verifyBatch: internal.rebuild.verifyBatch, + purgeBatch: internal.rebuild.purgeBatch, + fillTenantsBatch: internal.rebuild.fillTenantsBatch, + }, + }; const args = { generationId: id, operator: " " }; const calls = [ () => From 7d269db4a1f8441acb84a861fedcc2590c1fb188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:50:03 +0200 Subject: [PATCH 092/155] build: 3b fold-in the online rebuild asserts no live command or parent query on the progress row and records the entries' and batches' reruns there Co-Authored-By: Claude Opus 5.5 --- ...ld.online-rebuild-interrupt-resume.test.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts index 80e10f9..a5718d7 100644 --- a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts +++ b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts @@ -331,9 +331,18 @@ async function assertLog(w: World) { record.identifier === "rebuild:backfillBatch" && !record.willRetry, ).length, ).toBeGreaterThanOrEqual(350); - // rebuild.online-rebuild-interrupt-resume.sdp.md:36: no completion names progress as its conflict table. + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: no PlaceOrder, CancelOrder or parent query names progress + // as its conflict table; an operator entry and a batch may, and the run record counts them. + const live = new Set([ + getFunctionName(api.ordering.placeOrder), + getFunctionName(api.ordering.cancelOrder), + ]); expect( - log.filter((record) => record.occInfo?.tableName === "generationProgress"), + log.filter( + (record) => + record.occInfo?.tableName === "generationProgress" && + (live.has(record.identifier) || record.udfType === "Query"), + ), ).toEqual([]); const commands = new Set([ getFunctionName(api.ordering.placeOrder), @@ -473,6 +482,12 @@ bindExample( registryReruns: records(w).filter( (record) => record.occInfo?.tableName === "generations", ), + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: operator entries and batches that reran on the progress row. + progressReruns: records(w).filter( + (record) => + record.occInfo?.tableName === "generationProgress" && + record.identifier.startsWith("rebuild:"), + ), }, null, 2, From f84c3e5d0023d43c8501993f3f764ab13bf89116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:50:03 +0200 Subject: [PATCH 093/155] build: 3b fold-in the left-behind record counts the markers, progress and tenant-fill tables Co-Authored-By: Claude Opus 5.5 --- tests/native/place-order-attribution.ts | 3 +++ tests/native/place-order-contention.ts | 6 ++++++ tests/native/place-order-measurement.test.ts | 6 ++++++ tests/native/place-order-measurement.ts | 22 +++++++++++++++----- 4 files changed, 32 insertions(+), 5 deletions(-) diff --git a/tests/native/place-order-attribution.ts b/tests/native/place-order-attribution.ts index 51e4eb5..aa0eee4 100644 --- a/tests/native/place-order-attribution.ts +++ b/tests/native/place-order-attribution.ts @@ -26,6 +26,9 @@ export type OrderArgs = FunctionArgs; export const tables = { grants: ["grants", undefined], generations: ["generations", undefined], + progress: ["generationProgress", undefined], + tenantFill: ["tenantFill", undefined], + markers: ["projectionMarkers", undefined], receipts: ["receipts", undefined], summaries: ["orderSummaries", undefined], gates: ["maintenanceGates", undefined], diff --git a/tests/native/place-order-contention.ts b/tests/native/place-order-contention.ts index 60d53a0..1303690 100644 --- a/tests/native/place-order-contention.ts +++ b/tests/native/place-order-contention.ts @@ -38,6 +38,9 @@ export async function contentionCell(lines: 1 | 10 | 100, callers: 2 | 8 | 32) { expect(result.value.rowsBefore).toEqual({ grants: 4, generations: 1, + progress: 1, + tenantFill: 0, + markers: 0, receipts: 0, summaries: 0, orderStreams: 0, @@ -51,6 +54,9 @@ export async function contentionCell(lines: 1 | 10 | 100, callers: 2 | 8 | 32) { expect(result.value.rowsAfter).toEqual({ grants: 4, generations: 1, + progress: 1, + tenantFill: 0, + markers: 0, receipts: 1, summaries: 1, orderStreams: 1, diff --git a/tests/native/place-order-measurement.test.ts b/tests/native/place-order-measurement.test.ts index 0d5891b..9e4031a 100644 --- a/tests/native/place-order-measurement.test.ts +++ b/tests/native/place-order-measurement.test.ts @@ -48,6 +48,9 @@ test.each([1, 10, 100] as const)( expect(result.value.rowsAfter).toEqual({ grants: 4, generations: 1, + progress: 1, + tenantFill: 0, + markers: 0, receipts: 1, summaries: 1, orderStreams: 1, @@ -209,6 +212,9 @@ test.each([ expect(result.value.rowsAfter).toEqual({ grants: separateTenants ? 32 : 4, generations: 1, + progress: 1, + tenantFill: 0, + markers: 0, receipts: 8, summaries: 8, orderStreams: 8, diff --git a/tests/native/place-order-measurement.ts b/tests/native/place-order-measurement.ts index be300c3..b81c2b9 100644 --- a/tests/native/place-order-measurement.ts +++ b/tests/native/place-order-measurement.ts @@ -202,23 +202,33 @@ export async function schedules(backend: Backend) { return Object.fromEntries(entries); } // Rows left behind are counted over every table the production composition deploys that holds -// neither state, events, receipts nor read-model rows: the registry, the gate and both audit tables. -// Grants and the tenant list are setup. Markers are reported as not deployed while neither schema -// declares a table for them; a table either schema adds fails this check until it is counted. +// neither state, events, receipts nor read-model rows: the markers, the registry's generations, +// progress and tenant-fill rows, the gate and both audit tables (first-experiment.sdp.md:113). +// Grants and the tenant list are setup. A table either schema adds fails this check until it is counted. export function leftBehind(before: Stored, after: Stored) { expect(Object.keys(schema.tables).sort()).toEqual([ "auditRecords", + "generationProgress", "generations", "grants", "maintenanceGates", "operatorAudit", "orderSummaries", + "projectionMarkers", "receipts", + "tenantFill", "tenants", ]); expect(Object.keys(contextTables).sort()).toEqual(["events", "streams"]); const counted = ( - name: "generations" | "gates" | "auditRecords" | "operatorAudit", + name: + | "markers" + | "generations" + | "progress" + | "tenantFill" + | "gates" + | "auditRecords" + | "operatorAudit", ) => ({ before: before[name].length, after: after[name].length, @@ -227,8 +237,10 @@ export function leftBehind(before: Stored, after: Stored) { ).length, }); return { + markers: counted("markers"), generations: counted("generations"), - markers: "not deployed", + progress: counted("progress"), + tenantFill: counted("tenantFill"), gate: counted("gates"), audit: counted("auditRecords"), operatorAudit: counted("operatorAudit"), From 7b7097f74257510a94d2f818d1c75433ac317c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:53:32 +0200 Subject: [PATCH 094/155] design: 3b the observations after the slimmer row At the native tier, alone on the machine: the read of the active generation row is 95 bytes smaller now that the row carries no batch counts, 190 with a building generation, and each order summary row writes 9 bytes more for key in its by_status entry. The observed bytes in the first experiment and the tests that pin them follow. The measurement counts the scheduled rows its commands added, apart from the install's batches. Co-Authored-By: Claude Opus 5.5 --- .../specs/application/first-experiment.sdp.md | 6 +-- tests/native/place-order-attribution.test.ts | 32 ++++++++-------- tests/native/place-order-measurement.ts | 38 +++++++++++++------ tests/native/place-order-timing.test.ts | 6 +-- 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 00cfc1c..a3acd76 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -119,12 +119,12 @@ The experiment is an application, a test suite and a report. The application is - limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate document, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer; the gate read of the pipeline's step 7 runs in every one of these commands, and with no gate document in the table it finds nothing and adds no document read and no byte, so the counts above include it, and no measurement here has a gate document in the table (E-47, F13, E-8) - observedMeasurementPins: [extension] the measurement records name the native backend tier, the local backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124, `convex-test` 0.0.60, Node v24.19.0 and Linux 7.2.7-200.fc44.x86_64 on x64; these observations concern that local environment only (E-47, E-15, Sc L2-9) -- observedMeasurementBytes: [extension] at the native backend tier with `observedMeasurementPins`, fixed seven-character stock and order IDs, four grants and one active generation, orders of 1, 10 and 100 distinct stock items received once read 2943, 14454 and 129564 bytes and write 4271, 18410 and 159800 bytes; the corresponding write-index rows are 25, 97 and 817; a ten-line authorized duplicate or changed-input conflict reads 5 documents and 3441 bytes and writes zero, while short stock or a missing journal tail on the last line reads 32 documents and 13106 bytes and writes zero (E-47, F13) +- observedMeasurementBytes: [extension] at the native backend tier with `observedMeasurementPins`, fixed seven-character stock and order IDs, four grants and one active generation, orders of 1, 10 and 100 distinct stock items received once read 2848, 14359 and 129469 bytes and write 4280, 18419 and 159809 bytes; the corresponding write-index rows are 25, 97 and 817; a ten-line authorized duplicate or changed-input conflict reads 5 documents and 3441 bytes and writes zero, while short stock or a missing journal tail on the last line reads 32 documents and 13106 bytes and writes zero; the read of the active generation's `generations` row, which no longer carries the batch counts, is 95 bytes smaller than before, and each order summary row writes 9 bytes more for `key` in its `by_status` entry (E-47, F13) - observedContentionCosts: [extension] at the native backend tier with `observedMeasurementPins`, each of the nine cells at 1, 10 and 100 lines and 2, 8 and 32 callers reached one applied outcome and all remaining short-stock rejections with zero engine failures; an execution refused at commit read the healthy path's documents and bytes and wrote zero, and its final short-stock rejection, with the shared item first, read 5 documents and 1595 bytes and wrote zero; `occInfo` named Inventory's events or streams table across the cells and its `retryCount` was zero, one refused execution per rejected command, and the rerun count remains an observation per record (E-47, F1) -- observedUnrelatedStock: [extension] at the native backend tier with `observedMeasurementPins`, eight simultaneous ten-line commands on disjoint stock in one tenant and in eight unrelated tenants each recorded 8 executions, 8 applied outcomes and zero engine reruns; each command read 35 documents and 14454 bytes and wrote 24 documents and 18410 bytes, and the shared generation row was unchanged (E-47, F1) +- observedUnrelatedStock: [extension] at the native backend tier with `observedMeasurementPins`, eight simultaneous ten-line commands on disjoint stock in one tenant and in eight unrelated tenants each recorded 8 executions, 8 applied outcomes and zero engine reruns; each command read 35 documents and 14359 bytes and wrote 24 documents and 18419 bytes, the read of the shared generation row 95 bytes smaller than before, and the shared generation row was unchanged (E-47, F1) - observedLatency: [extension] on a quiet ThinkPad laptop running nothing else, at the native backend tier with `observedMeasurementPins` and the backend binary of sha256 `7ebc6a4696499640c2f2255395990b4f8b901b045388a0ced2ac765b81ebadae`, the latency cell of `npm run measure:place-order -- --quiet` over 30 accepted `PlaceOrder` commands per size after 5 discarded read a median of 20.3 ms and a 95th percentile of 21.3 ms at 1 line, 35.1 ms and 52.4 ms at 10 lines, and 166.3 ms and 178.3 ms at 100 lines, so every latency target `targetLatency` states is held; these are one local backend's times and say nothing of a hosted deployment (E-47, First experiment, Sc L2-9) - observedThroughput: [extension] with the same machine and pins, the throughput cell's 8 callers placing ten-line orders on disjoint stock completed 864 applied orders in 10.06 seconds, 85.9 orders a second, with zero engine reruns and no other outcome, beside the local backend's write cap of 4,194,304 bytes a second, so the target of 20 orders a second `targetThroughput` states is held (E-47, First experiment, F13) -- observedStockFoldGap: [extension] at the native backend tier with `observedMeasurementPins`, after `ReceiveStock` recorded quantity two, an admin change of current on-hand stock to three with the same stream version and journal tail let a one-line order allocate three, reading 8 documents and 2943 bytes and writing 6 documents and 4271 bytes; matching the tail version does not establish equality of current state with the complete event fold (E-47, D3) +- observedStockFoldGap: [extension] at the native backend tier with `observedMeasurementPins`, after `ReceiveStock` recorded quantity two, an admin change of current on-hand stock to three with the same stream version and journal tail let a one-line order allocate three, reading 8 documents and 2848 bytes and writing 6 documents and 4280 bytes; matching the tail version does not establish equality of current state with the complete event fold (E-47, D3) - streamBudgets: the stock item stream declares `budgetBytes` of 16 KiB and the order stream keeps the default 256 KiB, as `spec:application.orders-inventory-example` pins, so the allocation call's byte bound, 8 MiB over the planned streams' budgets, admits 512 stock streams and the adapter's count bound of `min(maxStreams, 256)` admits the 100 of the maximum order with room, while the single order stream costs at most 256 KiB; at the default budget on both stream types the byte bound would admit only 32 stock streams and the maximum run would be rejected `operationTooLarge`, which is why the budget is pinned here and the run at the maximum measures it (E-2, E-22, E-46, F13, Sc L2-3) - productDecisions: the maximum order size and the speed targets of `targetLatency`, `targetThroughput` and `targetContention` are set before the benchmark and recorded with the run; the speed targets are set for the pinned local backend on the machine of the run and say nothing of a hosted deployment; a target that is missed is reported as not held with its number, and no target is changed after the run (First experiment, OQ3) - targetLatency: without contention, latency is the execution time of the top-level mutation as the backend's function log reports it, taken over at least 30 `PlaceOrder` operations of each size after 5 discarded; the median is at most 100 ms at 1 line, 100 ms at 10 lines and 400 ms at 100 lines, and the 95th percentile at most 200 ms, 200 ms and 800 ms (First experiment, E-47) diff --git a/tests/native/place-order-attribution.test.ts b/tests/native/place-order-attribution.test.ts index 4dd859b..854c0b7 100644 --- a/tests/native/place-order-attribution.test.ts +++ b/tests/native/place-order-attribution.test.ts @@ -29,36 +29,36 @@ void anchor; const comparisons = [ { name: "healthy ten distinct stock items", - readBytes: 14454, - writeBytes: 18410, + readBytes: 14359, + writeBytes: 18419, reads: 35, writes: 24, }, { name: "one grant", - readBytes: 13536, - writeBytes: 18410, + readBytes: 13441, + writeBytes: 18419, reads: 32, writes: 24, }, { name: "two prior events on the last stock item", - readBytes: 14454, - writeBytes: 18410, + readBytes: 14359, + writeBytes: 18419, reads: 35, writes: 24, }, { name: "two lines on one stock item", - readBytes: 13175, - writeBytes: 16959, + readBytes: 13080, + writeBytes: 16968, reads: 32, writes: 22, }, { name: "no request key", - readBytes: 14454, - writeBytes: 15949, + readBytes: 14359, + writeBytes: 15958, reads: 35, writes: 23, }, @@ -119,22 +119,22 @@ const comparisons = [ }, { name: "stock row at version zero without events", - readBytes: 13853, - writeBytes: 18410, + readBytes: 13758, + writeBytes: 18419, reads: 34, writes: 24, }, { name: "existing summary row", - readBytes: 14996, - writeBytes: 18410, + readBytes: 14901, + writeBytes: 18419, reads: 37, writes: 24, }, { name: "building generation", - readBytes: 14900, - writeBytes: 18941, + readBytes: 14710, + writeBytes: 18959, reads: 36, writes: 25, }, diff --git a/tests/native/place-order-measurement.ts b/tests/native/place-order-measurement.ts index b81c2b9..b2f254b 100644 --- a/tests/native/place-order-measurement.ts +++ b/tests/native/place-order-measurement.ts @@ -184,23 +184,38 @@ export function pins(backend: Backend) { operatingSystem: `${platform()} ${release()} ${arch()}`, }; } -export async function schedules(backend: Backend) { +// The scheduled rows of each mount, the parent's included. Given an earlier read, only the rows +// added since it: a setup that installs the read model leaves its batches' rows behind. +export async function scheduledByMount(backend: Backend) { const entries = await Promise.all( [undefined, "orders", "inventory"].map( async (component) => [ component ?? "parent", - ( - await backend.admin.readTable( - "_scheduled_functions", - component === undefined ? {} : { component }, - ) - ).length, + await backend.admin.readTable( + "_scheduled_functions", + component === undefined ? {} : { component }, + ), ] as const, ), ); return Object.fromEntries(entries); } +export async function schedules( + backend: Backend, + before?: Awaited>, +) { + const now = await scheduledByMount(backend); + return Object.fromEntries( + Object.entries(now).map(([mount, rows]) => [ + mount, + rows.filter( + (row) => + !(before?.[mount] ?? []).some((prior) => prior._id === row._id), + ).length, + ]), + ); +} // Rows left behind are counted over every table the production composition deploys that holds // neither state, events, receipts nor read-model rows: the markers, the registry's generations, // progress and tenant-fill rows, the gate and both audit tables (first-experiment.sdp.md:113). @@ -415,6 +430,7 @@ export async function measured( ) { const backend = required(world.backend, "backend"); const before = await stored(backend); + const scheduledBefore = await scheduledByMount(backend); const mark = await backend.admin.logMark(); const answers = await Promise.all(args.map((arg) => send(world, arg))); // Close the interval with ordinary essential reads, one per tenant, before inspecting tables. @@ -448,7 +464,7 @@ export async function measured( 30000, ); const after = await stored(backend); - const scheduled = await schedules(backend); + const scheduled = await schedules(backend, scheduledBefore); const others = records.filter( (record) => !isOwn(record) && record.identifier !== "readModels:listOrderSummaries", @@ -508,9 +524,9 @@ export function expectUsage( }); } export const healthyUsage = { - 1: [8, 6, 2943, 4271], - 10: [35, 24, 14454, 18410], - 100: [305, 204, 129564, 159800], + 1: [8, 6, 2848, 4280], + 10: [35, 24, 14359, 18419], + 100: [305, 204, 129469, 159809], } as const; export type PlaceOrderMeasurement = ReturnType; diff --git a/tests/native/place-order-timing.test.ts b/tests/native/place-order-timing.test.ts index 3046300..183225d 100644 --- a/tests/native/place-order-timing.test.ts +++ b/tests/native/place-order-timing.test.ts @@ -199,15 +199,15 @@ timing( )) { expect(usageOf(record).databaseReadDocuments).toBe(35); expect(usageOf(record).databaseWriteDocuments).toBe(24); - expect(usageOf(record).databaseWriteBytes).toBe(18410); + expect(usageOf(record).databaseWriteBytes).toBe(18419); expect(usageOf(record).databaseWriteIndexRows).toBe(97); // The first order reads StockReceived; later orders read the larger StockAllocated tail. - expect([14454, 14614]).toContain(usageOf(record).databaseReadBytes); + expect([14359, 14519]).toContain(usageOf(record).databaseReadBytes); } expect( finals(collected.records).filter( (record) => - record.error === null && usageOf(record).databaseReadBytes === 14454, + record.error === null && usageOf(record).databaseReadBytes === 14359, ), ).toHaveLength(8); expect(value.rowsLeftBehind.generations.created).toBe(0); From 17d7b4e3469f3268c7d7d2bc7a466cf7a83df37f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:54:18 +0200 Subject: [PATCH 095/155] build: 3b fold-in rollbackGeneration refuses while the tenant list is being filled A rollback reopens a generation, so it and a fill exclude each other as a start and a fill do; the refusal follows the pause check and precedes the in-flight check. Co-Authored-By: Claude Opus 5.5 --- design/specs/application/rebuild.sdp.md | 4 ++-- src/read-model/rebuild.ts | 4 ++++ tests/simulator/rebuild.test.ts | 29 +++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 40e8ea2..68a521e 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -81,7 +81,7 @@ The registry, the progress row, the per-entity marker and the gate document are - rule: [extension] A tenant whose row enters the tenant list behind a backfill's cursor is not visited by that pass and needs no visit, because every stream it owns was created after the generation was registered and so was written live; the verify pass starts again at the first tenant and visits it (E-49, E-8, D9) - rule: [extension] `startGeneration` refuses when the tenant list holds no row while the `grants` table holds one, with a message that names `fillTenants`, because a generation started then would visit no tenant, end its passes and be switched with no row of any tenant (E-49, E-8, Law 11) - rule: [extension] `fillTenants` is an operator entry of `spec:command.actor-and-scope` that fills the tenant list from the `grants` table: its batches walk the grants in tenant-ID order, one distinct tenant per read, at most `limitTenantFillBatch` tenants per batch, through the `tenantFill` progress row of their own, and insert a tenant's row only where none exists, so each tenant is inserted once however often the walk runs; it reads a grant row only as the record that its tenant exists and never to authorize, and it writes nothing but tenant rows and its progress row (E-49, E-48, E-40, D11, Law 11) -- rule: [extension] A tenant fill and a generation in flight exclude each other: `fillTenants` refuses while a read model of the deployment has a generation in `building`, `verifying` or `verified`, and `startGeneration` refuses while the fill's pass is `fill`, so no tenant whose streams predate a generation enters the list behind that generation's cursor (E-49, E-8, D9, F1) +- rule: [extension] A tenant fill and a generation in flight exclude each other: `fillTenants` refuses while a read model of the deployment has a generation in `building`, `verifying` or `verified`, and `startGeneration` and `rollbackGeneration`, which reopens a generation, refuse while the fill's pass is `fill`, so no tenant whose streams predate a generation enters the list behind that generation's cursor (E-49, E-8, D9, F1) - rule: A backfill batch racing a live command is ordered by optimistic concurrency; the batch's read-then-write of a row is one mutation, so the batch never overwrites a row a command wrote after the batch read its source (D9, F1) - rule: [extension] Every batch, whether its source is a context's enumeration or a parent table, is driven by a self-scheduled internal mutation under one batch contract, and the rebuild uses `@convex-dev/migrations` for nothing (E-40, F17, F8) - rule: [extension] A projection version change is a new generation, never an in-place rewrite of the active one (E-8, D8) @@ -131,7 +131,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to no `lastError` and `updatedAt` and leaving its `batchSize`, cursor and counts as they are, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) - fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain`; it takes no batch size, so a resumed pass runs at the progress row's `batchSize`, a purge capped by `limitPurgeBatch`; for a generation in any state whose progress row names a pass, so it continues a backfill or verify pass of a generation in `building` or `verifying` and a started purge of a `retired` or `aborted` one, which is how an operator continues a purge a restore stranded (D9, Sc L2-5, E-40, E-48) - fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) -- fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `pass` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, the in-flight error of `startGeneration`, naming that generation's number and state, when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, no `lastError` and `updatedAt`, with `batchSize` and the other counts kept, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) +- fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `pass` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, `"The tenant list is being filled; rollbackGeneration waits until fillTenants has ended"` while the `tenantFill` row's `pass` is `fill`, the in-flight error of `startGeneration`, naming that generation's number and state, when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, no `lastError` and `updatedAt`, with `batchSize` and the other counts kept, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) - fnAbortGeneration: `export const abortGeneration = internalMutation({ args: { generationId: v.id("generations"), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and `"Generation " + n + " of " + name + " is " + state + "; abortGeneration needs building, verifying or verified"` unless the state is one of the three; otherwise patching the row to `state: "aborted"`, fence plus 1, `changedAt` and `changedBy`, patching the progress row to `pass: "idle"`, `lastError: "aborted: " + reason` and `updatedAt`, with the cursor and the counts kept, and calling the write pause's `resumeScope` for every closed entry that names this generation (D9, Sc L2-6, E-8, E-48) - fnPurgeGeneration: `export const purgeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; purgeGeneration needs retired or aborted"` unless the state is one of the two, `"Generation " + n + " of " + name + " is inside its rollback period"` for a retired generation while the mutation's `Date.now()` is before `retireAfter`, and `"Generation " + n + " of " + name + " is already being purged; resumeGeneration continues it"` when `pass` is `purge`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to `pass: "purge"`, `cursor: null`, no `lastError` and `updatedAt`, and scheduling `refs.purgeBatch` with the new fence (D19, E-40, E-48) - fnGetGenerations: `export const getGenerations = internalQuery({ args: { readModel: v.string() }, returns: v.array(v.object({ generation: generationDoc, progress: progressDoc })), handler })` the newest `limitGenerationsListed` generations of the read model by `by_read_model` in descending order, each with its progress row, where `generationDoc` and `progressDoc` are the two tables' document validators with `_id` and `_creationTime`, and `[]` for a read model the composition does not declare; an operator entry that only reads, so it takes no operator (E-8, E-40, E-48, D19) diff --git a/src/read-model/rebuild.ts b/src/read-model/rebuild.ts index 3a8a241..8f6f92c 100644 --- a/src/read-model/rebuild.ts +++ b/src/read-model/rebuild.ts @@ -355,6 +355,10 @@ export async function rollbackGeneration( throw new Error( `${description(generation)} needs a write pause and cannot be rolled back`, ); + if ((await ctx.db.query("tenantFill").first())?.pass === "fill") + throw new Error( + "The tenant list is being filled; rollbackGeneration waits until fillTenants has ended", + ); const flight = await inFlight(ctx, generation.readModel); if (flight !== undefined) throw flightError(flight); projectionOf( diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts index 3a70409..680cd57 100644 --- a/tests/simulator/rebuild.test.ts +++ b/tests/simulator/rebuild.test.ts @@ -1274,6 +1274,35 @@ test("convex-test: rollback checks period, purge, pause, in-flight generation an ); await patchGeneration(t, old, { pauseRequired: false }); const other = await start(t); + // rebuild.sdp.md:84: a fill and a reopened generation exclude each other; the fill refusal + // precedes the in-flight refusal. + const fillRow = { + fence: 1, + cursor: null, + batchesDone: 0, + tenantsRead: 0, + tenantsInserted: 0, + startedAt: Date.now(), + startedBy: operator, + changedAt: Date.now(), + changedBy: operator, + updatedAt: Date.now(), + }; + const fillId = await t.run(async (ctx) => { + const prior = await ctx.db.query("tenantFill").first(); + if (prior !== null) await ctx.db.delete(prior._id); + return ctx.db.insert("tenantFill", { ...fillRow, pass: "fill" }); + }); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }), + "The tenant list is being filled; rollbackGeneration waits until fillTenants has ended", + ); + await t.run((ctx) => ctx.db.patch(fillId, { pass: "idle" })); for (const state of ["building", "verifying", "verified"] as const) { await patchGeneration(t, other, { state }); await refused( From bd247875af80b078d6d50f499ce3a6a0be34eab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:54:28 +0200 Subject: [PATCH 096/155] design: 3b the tenant fill copies the IDs the grants hold Co-Authored-By: Claude Opus 5.5 --- design/specs/application/rebuild.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 68a521e..7427e09 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -138,7 +138,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - tableTenantFill: `tenantFill: defineTable({ pass: v.union(v.literal("fill"), v.literal("idle")), fence: v.number(), cursor: v.union(v.string(), v.null()), batchesDone: v.number(), tenantsRead: v.number(), tenantsInserted: v.number(), lastError: v.optional(v.string()), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), updatedAt: v.number() })` in `readModelTables`, at most one row per deployment, the progress row of the tenant fill and nothing else's; operational data of the deployment with no tenant dimension and no index, read with `ctx.db.query("tenantFill").first()`; `cursor` is the last tenant ID the walk passed, `null` before its first, `tenantsRead` counts the tenants the walk found and `tenantsInserted` the rows it inserted (E-49, E-40, E-48) - fnFillTenants: `export const fillTenants = internalMutation({ args: { operator: v.string() }, returns: v.null(), handler })` in order: `assertOperator`; the restore door, throwing `"fillTenants is refused while the restore door is closed"` when `process.env.MAINTENANCE_MODE` is exactly `restore`, because the fill writes the tenant list that a restore replaces, and no other read of the gate, because the fill writes no tenant's data and so no closed tenant scope stops it; for each target of `config.targets` the in-flight check by `by_read_model_state`, throwing `"Read model " + name + " has generation " + n + " in " + state + "; fillTenants needs no generation in flight"`; then, with `now` the mutation's `Date.now()`, when no `tenantFill` row exists it inserts one with `pass: "fill"`, `fence: 1`, `cursor: null`, every count 0, `startedAt`, `changedAt` and `updatedAt` at `now` and `startedBy` and `changedBy` the operator, and when one exists it patches it to `pass: "fill"`, fence plus 1, the cursor and the counts kept when the pass was `fill` and `null` and 0 when it was `idle`, no `lastError`, `changedAt`, `changedBy` and `updatedAt`; then it schedules `refs.fillTenantsBatch` with the row's fence through `ctx.scheduler.runAfter(0, ...)`; so a call continues a fill whose chain a restore, an error or a closed gate ended, and a call after a finished fill walks the grants again and inserts only what is missing (E-49, E-48, E-40, F8) - fnFillTenantsBatch: `export const fillTenantsBatch = internalMutation({ args: { fence: v.number() }, returns: v.null(), handler })` (E-49, E-40) -- fillSteps: `fillTenantsBatch` loads the `tenantFill` row and returns without writing when it is absent, `pass` is not `fill` or `fence` differs from its argument; it calls `gateAllows(ctx, ["all"])` and, when the answer is not allowed, parks as step 3 does, patching `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt` and scheduling nothing, so the fill waits for `fillTenants` after the gate reopens; otherwise it repeats at most `limitTenantFillBatch` times: it reads `ctx.db.query("grants").withIndex("by_principal", (q) => (cursor === null ? q : q.gt("tenantId", cursor))).first()`, stops when that answers no row, and otherwise reads `tenants` by `by_tenant` for the grant's `tenantId`, inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, adds one to `tenantsRead` and, for an insert, one to `tenantsInserted`, and sets the cursor to that `tenantId`; a batch that took `limitTenantFillBatch` tenants patches the row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt` and schedules `refs.fillTenantsBatch` with the same fence; a batch that found no further grant patches the row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt` and schedules nothing (E-49, E-40, F8, Law 11) +- fillSteps: `fillTenantsBatch` loads the `tenantFill` row and returns without writing when it is absent, `pass` is not `fill` or `fence` differs from its argument; it calls `gateAllows(ctx, ["all"])` and, when the answer is not allowed, parks as step 3 does, patching `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt` and scheduling nothing, so the fill waits for `fillTenants` after the gate reopens; otherwise it repeats at most `limitTenantFillBatch` times: it reads `ctx.db.query("grants").withIndex("by_principal", (q) => (cursor === null ? q : q.gt("tenantId", cursor))).first()`, stops when that answers no row, and otherwise reads `tenants` by `by_tenant` for the grant's `tenantId`, inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, adds one to `tenantsRead` and, for an insert, one to `tenantsInserted`, and sets the cursor to that `tenantId`; a batch that took `limitTenantFillBatch` tenants patches the row with the cursor, the counts, `batchesDone` plus 1, no `lastError` and `updatedAt` and schedules `refs.fillTenantsBatch` with the same fence; a batch that found no further grant patches the row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt` and schedules nothing; the fill copies each tenant ID as the grant holds it, so a grant written before the tenant-ID bound was enforced is the operator's to repair before the fill, and the first experiment holds none (E-49, E-40, F8, Law 11) - limitTenantFillBatch: 8 tenants per fill batch, a library constant; a grant row's bytes are bounded only by the 1 MiB document limit, so a batch reads at most 8 grant rows, 8 MiB, one per tenant through `by_principal`, at most 8 tenant rows of at most `limitIdLength` bytes of tenant ID each, the gate document and the `tenantFill` row, and writes at most 8 tenant rows and the `tenantFill` row, under the 16 MiB read and write ceilings (E-49, E-40, F13) - errorUnknownGeneration: every entry and every helper that takes a `generationId` throws `"No generation " + generationId` when the row is absent, and a batch returns without writing instead (E-8) - stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its pass is not `idle`; `resumeGeneration` takes no batch size, so the operator answers a stalled backfill or verify pass with `abortGeneration` and a new `startGeneration` with a smaller `batchSize`, down to 1, and a purge batch deletes rows of a generation nothing else writes (E-40, F1, F9, Probe 6) From 48997520271edea61b6aab28496dd282dca65709 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:56:56 +0200 Subject: [PATCH 097/155] build: 3b fold-in the install's one batch, its rows and markers, and the stated version are asserted startGeneration schedules exactly one batch, counted among the scheduler's rows; an install leaves one row per subject and no marker; the shared install helpers assert the active generation is at the version they stated. Co-Authored-By: Claude Opus 5.5 --- tests/native/rebuild-install.ts | 8 ++++++-- tests/simulator/gate-support.ts | 16 ++++++++++++++-- tests/simulator/rebuild.test.ts | 30 ++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/tests/native/rebuild-install.ts b/tests/native/rebuild-install.ts index cf8c6e1..e64309b 100644 --- a/tests/native/rebuild-install.ts +++ b/tests/native/rebuild-install.ts @@ -1,6 +1,7 @@ import { getFunctionName } from "convex/server"; import type { Value } from "convex/values"; import { setTimeout as sleep } from "node:timers/promises"; +import { expect } from "vitest"; import { internal as production } from "../../example/convex/_generated/api.js"; import { internal as fixture } from "../../fixture/convex/_generated/api.js"; import type { Backend } from "../../harness/backend.js"; @@ -71,14 +72,17 @@ async function install( refs: { start: string; switch: string }, operator: string, ) { + const projectionVersion = 1; const id = await backend.admin.run(refs.start, { readModel, - projectionVersion: 1, + projectionVersion, operator, }); if (typeof id !== "string") throw new Error("startGeneration did not return an ID"); - await waitForGeneration(backend, readModel, id); + const verified = await waitForGeneration(backend, readModel, id); + // rebuild.sdp.md:115: the installed generation is at the version the setup stated. + expect(verified.generation.projectionVersion).toBe(projectionVersion); await backend.admin.run(refs.switch, { generationId: id, operator }); return id; } diff --git a/tests/simulator/gate-support.ts b/tests/simulator/gate-support.ts index d3e9bd0..8b3d2e7 100644 --- a/tests/simulator/gate-support.ts +++ b/tests/simulator/gate-support.ts @@ -127,9 +127,10 @@ export async function installFixtureReadModel( if (ownedTimers) vi.useFakeTimers(); try { // native-harness.sdp.md:90: documentTitle setups select version 1. + const projectionVersion = 1; const generationId = await t.mutation(internal.rebuild.startGeneration, { readModel, - projectionVersion: 1, + projectionVersion, operator, }); await t.finishAllScheduledFunctions(() => vi.runOnlyPendingTimers(), 1000); @@ -137,6 +138,11 @@ export async function installFixtureReadModel( generationId, operator, }); + // rebuild.sdp.md:115: the active generation is at the version the setup stated. + expect(await t.run((ctx) => ctx.db.get(generationId))).toMatchObject({ + state: "active", + projectionVersion, + }); return generationId; } finally { if (ownedTimers) vi.useRealTimers(); @@ -149,9 +155,10 @@ export async function installOrderSummary( const ownedTimers = !vi.isFakeTimers(); if (ownedTimers) vi.useFakeTimers(); try { + const projectionVersion = 1; const generationId = await t.mutation(production.rebuild.startGeneration, { readModel: "orderSummary", - projectionVersion: 1, + projectionVersion, operator, }); await t.finishAllScheduledFunctions(() => vi.runOnlyPendingTimers(), 1000); @@ -159,6 +166,11 @@ export async function installOrderSummary( generationId, operator, }); + // rebuild.sdp.md:115: the active generation is at the version the setup stated. + expect(await t.run((ctx) => ctx.db.get(generationId))).toMatchObject({ + state: "active", + projectionVersion, + }); return generationId; } finally { if (ownedTimers) vi.useRealTimers(); diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts index 680cd57..353c5d7 100644 --- a/tests/simulator/rebuild.test.ts +++ b/tests/simulator/rebuild.test.ts @@ -2317,3 +2317,33 @@ test("convex-test: a failed activation commits neither half of the switch", asyn expect((await read(t, old)).generation.state).toBe("active"); expect((await read(t, id)).generation.state).toBe("verified"); }); + +// rebuild.sdp.md fnStartGeneration:115, Design:107; projection-contract.sdp.md readModelRowShape. +test("convex-test: startGeneration schedules exactly one batch and an install leaves one row per subject and no marker", async () => { + const t = rebuildApp(); + for (const documentId of ["a1", "a2", "a3"]) await create(t, documentId); + const before = await scheduled(t); + const id = await start(t); + const added = (await scheduled(t)).filter( + (row) => !before.some((prior) => prior._id === row._id), + ); + expect(added.map(({ name, args }) => ({ name, args }))).toStrictEqual([ + { name: "rebuild:backfillBatch", args: [{ generationId: id, fence: 1 }] }, + ]); + await finish(t, id); + await t.mutation(internal.rebuild.switchGeneration, { + generationId: id, + operator, + }); + const generation = (await read(t, id)).generation.generation; + expect( + (await rows(t)) + .filter((row) => row.generation === generation) + .map(({ key }) => key) + .sort(), + ).toStrictEqual(["a1", "a2", "a3"]); + // documentTitle is a per-entity read model: markers belong to aggregate rows, so none is written. + expect( + await t.run((ctx) => ctx.db.query("projectionMarkers").collect()), + ).toStrictEqual([]); +}); From 334822a369a19feb80fb0c7daed6f0a29c355c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:58:12 +0200 Subject: [PATCH 098/155] build: 3b fold-in the online rebuild counts its batches against the progress row The backfill and verify completions that wrote number batchesDone, and each pass takes at least the seeded orders and at most the orders at verified plus one per tenant; the example's opening bullet says so. Co-Authored-By: Claude Opus 5.5 --- ...ild.online-rebuild-interrupt-resume.sdp.md | 2 +- ...ld.online-rebuild-interrupt-resume.test.ts | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md index 86a1bed..27b01a3 100644 --- a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md +++ b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md @@ -30,7 +30,7 @@ And writes {writes: "never stopped"} ## Verification — executable - Runs in the native tier; every test owns its disposable backend. -- The test, in a file whose own timeout is 300 s, installs the order summary's generation 1 through its first rebuild, seeds 350 orders across two tenants, 250 in the first in tenant-ID order and 100 in the second, starts generation 2 with `projectionVersion` 1 stated, a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that `batchesDone` then stops advancing below 350 and that the cursor names a tenant; while a chain is expected to run, a progress row whose `batchesDone` has not advanced for 30 s fails the test with the generation row and the progress row in the message. +- The test, in a file whose own timeout is 300 s, installs the order summary's generation 1 through its first rebuild, seeds 350 orders across two tenants, 250 in the first in tenant-ID order and 100 in the second, starts generation 2 with `projectionVersion` 1 stated, a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that `batchesDone` then stops advancing below 350 and that the cursor names a tenant; while a chain is expected to run, a progress row whose `batchesDone` has not advanced for 30 s fails the test with the generation row and the progress row in the message; once generation 2 is `verified` it asserts that the backfill and verify completions in the function log that wrote number the progress row's `batchesDone`, and that each pass took at least the 350 seeded orders and at most the orders at `verified` plus one per tenant. - While the chain is interrupted the test sends `CancelOrder` to one of the second tenant's 100 seeded orders and to an order of the first tenant whose row generation 2 already holds, read from the table before the command; it asserts at once that the first tenant's order has its row in generation 2 `cancelled`, and, once generation 2 is `verified`, that the second tenant's order has its row in generation 2 `cancelled`, whichever pass wrote it. - After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that, once the generation is `verified`, every order the Orders context's `list` enumerates in either tenant, read with admin access and `includeDeleted`, has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, that `backfillBatch` run with admin access under the fence the interrupt replaced leaves the progress row unchanged, that the function log shows no action and no completion during the batches other than a batch, an operator entry or a live command, and that the journals, read by operation, hold the event and receipt counts the live commands produced. - The test asserts that no command of the run ends in a failure, that no completion record of a `PlaceOrder`, a `CancelOrder` or a parent query names `generationProgress` in its `occInfo`, and that the completion records whose `occInfo` names `generations` number at most the operator entries and pass ends of the run times the callers the test keeps in flight plus one for the chain, and records those reruns with their `occInfo` as a measurement, beside them the count of the reruns of operator entries and batches whose `occInfo` names `generationProgress`, each with its `occInfo`; it reads generation 2's row at the interrupt, after the resume and at `verified`, and asserts that the interrupt and the resume changed only `fence`, `changedAt` and `changedBy` and that from the resume to `verified` only `state` changed. diff --git a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts index a5718d7..47459cf 100644 --- a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts +++ b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts @@ -77,6 +77,7 @@ interface World { mark?: LogMark; batchStart?: number; batchEnd?: number; + subjectsAtVerified?: number; cursors: string[]; live: boolean; workers: Promise[]; @@ -207,6 +208,28 @@ async function verifyBatchCount(w: World, batchSize: number) { } return count; } +// The orders the Orders context's list enumerates in both tenants, deleted ones included. +async function subjects(w: World) { + let count = 0; + for (const tenantId of tenants) { + let cursor: string | null = null; + for (;;) { + const page = (await backendOf(w).admin.run( + "queries/order:list", + { + tenantId, + includeDeleted: true, + paginationOpts: { cursor, numItems: 100 }, + }, + { component: "orders" }, + )) as unknown as PaginationResult; + count += page.page.length; + if (page.isDone) break; + cursor = page.continueCursor; + } + } + return count; +} async function assertCoverage(w: World, generation: number) { const target = await rows(w, generation); const expected: { tenantId: string; key: string }[] = []; @@ -324,6 +347,29 @@ async function assertLog(w: World) { expect(during.filter((record) => !allowed.has(record.identifier))).toEqual( [], ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: the batches that wrote, by pass, are the progress + // row's batchesDone; a batch under a replaced fence completes without writing and is not counted. + const wrote = (identifier: string) => + during.filter( + (record) => + record.identifier === identifier && + !record.willRetry && + record.error === null && + (record.usageStats["databaseWriteDocuments"] ?? 0) > 0, + ).length; + const backfills = wrote("rebuild:backfillBatch"); + const verifies = wrote("rebuild:verifyBatch"); + expect(backfills + verifies).toBe( + required(w.verified, "verified generation").progress.batchesDone, + ); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33: at a batch size of 1 each pass takes one batch per + // subject it visits and at most one more per tenant. + for (const count of [backfills, verifies]) { + expect(count).toBeGreaterThanOrEqual(350); + expect(count).toBeLessThanOrEqual( + required(w.subjectsAtVerified, "subjects at verified") + tenants.length, + ); + } // rebuild.online-rebuild-interrupt-resume.sdp.md:35: the batch log assertion is not vacuous. expect( during.filter( @@ -722,6 +768,7 @@ bindExample( ); await stopLive(w); w.batchEnd = (await backendOf(w).admin.logMark()).cursorMs; + w.subjectsAtVerified = await subjects(w); // rebuild.online-rebuild-interrupt-resume.sdp.md:35: the backfill cursor visits the first tenant before the second. expect(w.cursors.indexOf(tenants[0])).toBeGreaterThanOrEqual(0); // rebuild.online-rebuild-interrupt-resume.sdp.md:35: the cursor crosses to the second tenant after resume. From c20ab045dbeaeb836b811579569bc4ca2611de84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:58:13 +0200 Subject: [PATCH 099/155] build: 3b fold-in the progress, tenant-fill and markers tables in every stored-row comparison Co-Authored-By: Claude Opus 5.5 --- tests/native/order-use-case.ts | 3 +++ tests/native/scheduler-composition.ts | 3 +++ tests/simulator/rebuild-support.ts | 1 + 3 files changed, 7 insertions(+) diff --git a/tests/native/order-use-case.ts b/tests/native/order-use-case.ts index d34184f..02bfcfc 100644 --- a/tests/native/order-use-case.ts +++ b/tests/native/order-use-case.ts @@ -81,6 +81,9 @@ export const line = ( const tables = [ { table: "receipts" }, { table: "generations" }, + { table: "generationProgress" }, + { table: "tenantFill" }, + { table: "projectionMarkers" }, { table: "orderSummaries" }, { table: "streams", component: "orders" }, { table: "events", component: "orders" }, diff --git a/tests/native/scheduler-composition.ts b/tests/native/scheduler-composition.ts index d1f6a2e..f565e25 100644 --- a/tests/native/scheduler-composition.ts +++ b/tests/native/scheduler-composition.ts @@ -118,6 +118,9 @@ export async function dataRows(backend: Backend) { "auditRecords", "operatorAudit", "generations", + "generationProgress", + "tenantFill", + "projectionMarkers", "orderSummaries", "schedulerData", "schedulerReferences", diff --git a/tests/simulator/rebuild-support.ts b/tests/simulator/rebuild-support.ts index 5a164bd..4c5b918 100644 --- a/tests/simulator/rebuild-support.ts +++ b/tests/simulator/rebuild-support.ts @@ -103,6 +103,7 @@ export async function snapshot(t: App) { summaries: await ctx.db.query("documentSummaries").collect(), tenants: await ctx.db.query("tenants").collect(), fills: await ctx.db.query("tenantFill").collect(), + markers: await ctx.db.query("projectionMarkers").collect(), scheduled: await ctx.db.system.query("_scheduled_functions").collect(), })); } From 4a7ef82d92e688e7372a7d6514d5cbd8e82c3e67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 04:59:34 +0200 Subject: [PATCH 100/155] design: 3b ledger and register Five ledger rows proposed partially-fixed with their reasons: the checkpoint on the row commands read, the tenant list, the batches bounded in documents only, and the layout's two version and absence rows. The register's five rows are left for the main thread to record. Co-Authored-By: Claude Opus 5.5 --- design/reviews/consensus-ledger.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index f3fc1fe..373963f 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -1916,8 +1916,8 @@ ], "issue": "A read model's binding carries one projection at one version: `ReadModelBinding = { name; version; source; projection }` (command-declaration typeReadModelBinding). The pipeline's step 9 reads `generationsToWrite`, which returns generation numbers only, and calls `applyProjection(ctx, projection, { ..., generations })` once with every generation to write (projection-contract fnApplyProjection, callers). So the active generation, built under the previous projection version, and the building one both receive the output of the one projection the deployed code holds. The contract says 'a new version is a new generation, never an in-place rewrite' and rebuild repeats it as a rule, yet from the deploy that changes a projection, before `startGeneration` is even called, every live command rewrites the active generation's rows in place with the new version's output and stamps the new `projectionVersion` on them. Queries then read a generation that mixes two row shapes until the switch. Rollback has the same hole from the other side: `rollbackGeneration` reopens the retired generation, 'live commands write it again', and they write it with the new projection, so the generation the operator rolls back to is no longer the old build. Sc L2-5's 'cutover and rollback work' holds only when the projection did not change, which is the one case the generation mechanism is not for. No Spec says which projection code writes which generation. A second gap sits beside it: `startGeneration`, the batches, the restore's inventory and its read-model check take a read model by name, but its projection and sources live only on command declarations, repeated on every command that maintains it, and no Spec says how a name resolves to them.", "fix": "Pin one read-model registry in the parent, one constant per read model: its name, its sources and its projections keyed by version. Command declarations list read-model names or reference the constant, so the version and the projection are stated once (D12). `generationsToWrite` returns `{ generation, projectionVersion }` pairs and step 9 writes each generation through the projection whose version the generation row records; the registration check fails when a generation in `active`, `building`, `verifying`, `verified` or `retired` inside its rollback period names a version the registry no longer holds. State the standing cost under E-8 or E-9: the previous projection stays in the code until its generation is purged. Make the L2-5 example rebuild with a changed projection version and assert that the active generation's rows keep the old version until the switch and that the rolled-back generation is written by the old projection.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-01: At HEAD, generation-registry.sdp.md:72 already returns each generation with \"projectionVersion\" and command-declaration.sdp.md:66 references one ReadModel, so the issue's old numbers-only and repeated-binding clauses are stale, but projection-contract.sdp.md:92 still stamps \"the declared projection's version\" on every row and can mix v1 and v2. Keep the version-selected projection work and old-version retention, adapting the fix to the existing ReadModel constant rather than repeating wiring that is already shared. The build must show: convex-test must keep active v1 and building v2 rows in their own shapes under live creates and updates, and a native backend rollout must keep reads valid and return to v1 after intervening writes. A guard-only cut must refuse mismatch before any write and state the supported deployment procedure. S4 on paper, 2026-10-02. `spec:application.projection-contract` pins `projections`, the latest first, and `projectionOf`, and its `step1` and `step4` write each generation with the projection whose version its row records; `spec:application.rebuild` refuses an undeclared version at `startGeneration` and `rollbackGeneration`, and the E-9 question states the standing cost. No check at registration is stated: a generation whose version the read model no longer declares fails at the write, through `projectionOf`. The build must show: at convex-test, an active generation of version 1 and a building one of version 2 each keep their own rows under live creates and updates; on the native backend, a rollout to version 2 and a return to version 1 after intervening writes.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-01: At HEAD, generation-registry.sdp.md:72 already returns each generation with \"projectionVersion\" and command-declaration.sdp.md:66 references one ReadModel, so the issue's old numbers-only and repeated-binding clauses are stale, but projection-contract.sdp.md:92 still stamps \"the declared projection's version\" on every row and can mix v1 and v2. Keep the version-selected projection work and old-version retention, adapting the fix to the existing ReadModel constant rather than repeating wiring that is already shared. The build must show: convex-test must keep active v1 and building v2 rows in their own shapes under live creates and updates, and a native backend rollout must keep reads valid and return to v1 after intervening writes. A guard-only cut must refuse mismatch before any write and state the supported deployment procedure. S4 on paper, 2026-10-02. `spec:application.projection-contract` pins `projections`, the latest first, and `projectionOf`, and its `step1` and `step4` write each generation with the projection whose version its row records; `spec:application.rebuild` refuses an undeclared version at `startGeneration` and `rollbackGeneration`, and the E-9 question states the standing cost. No check at registration is stated: a generation whose version the read model no longer declares fails at the write, through `projectionOf`. The build must show: at convex-test, an active generation of version 1 and a building one of version 2 each keep their own rows under live creates and updates; on the native backend, a rollout to version 2 and a return to version 1 after intervening writes. The online rebuild of the order summary: `applyProjection` writes each generation by its own projection version (projection-contract.sdp.md step1), with two versions at once proven at `convex-test` on the fixture's `documentTitle`. Partially fixed: the native L2-5 run builds generation 2 at version 1, so a native rollout to version 2 and back stays open.", "slice": "S4", "detector": "convex-test" }, @@ -2039,8 +2039,8 @@ ], "issue": "The round-2 ruling that a generation spans every tenant made the rebuild depend on 'the application's tenant list, read in tenant-ID order': `startGeneration` puts the cursor 'at the first tenant' of it, `tenantIteration` moves to 'the next tenant' of it, the restore's checks under `all` iterate it 'the way the rebuild's `tenantIteration` does', the baseline driver runs 'one chain per tenant' because 'the parent, not the component, knows the tenants', and the receipt sweep runs per tenant. No Spec defines that list. The parent's pinned tables are `grants`, `receipts`, `generations`, `projectionMarkers`, `maintenanceGates`, `imports`, `restoreRuns`, `auditRecords` and `personalData`; none enumerates tenants, and `grants` has no index that yields distinct tenant IDs. The L2-5 and L2-8 examples seed two tenants and assert the cursor crosses them, so the first experiment cannot build its rebuild without inventing the table, its index, and what happens to a tenant created while a pass is under way.", "fix": "Pin a `tenants` table in the parent on `spec:command.actor-and-scope` or `spec:command.tenancy-and-authority`: `tenantId`, `createdAt`, an index on `tenantId`, written by the command or operator function that creates a tenant, as an [extension] with its register row. State in `tenantIteration` that a pass reads it by index after the cursor's tenant ID, one tenant per step, and rule on a tenant created behind the cursor: on the online path its streams are all created after `startGeneration` and so written live, and the verify pass enumerates again; under a write pause no stream can be created in a closed source. Point the restore's scope, the baseline driver's `driverTenancy` and the receipt sweep at the same table.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-10: At HEAD, rebuild.sdp.md:103 requires \"the application's tenant list\" without declaring its authority or enumeration, so an all-tenant pass cannot locate its next tenant. The issue and structured enumeration or shared compare-helper fix remain right; completion must require every required check passed. The build must show: convex-test and native backend enumeration must visit every required tenant and exercise a tenant created behind the cursor under the ruled online or paused semantics, without treating absent tenant scope as a wildcard. S4 on paper, 2026-10-02. `spec:command.actor-and-scope` pins the `tenants` table, `insertGrant` writing a tenant's row and `nextTenant` (E-49); `spec:application.rebuild` `step2`, `step6` and the rule on a tenant behind the cursor iterate it, and `spec:application.restore` `validatorCheckScope` points at `nextTenant`. The receipt sweep of `spec:command.receipt-table` does not name it yet, for the unit that builds the sweep. The build must show at convex-test and on the native backend: two tenants and the cursor crossing, an empty tenant, and a tenant inserted behind the cursor.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-10: At HEAD, rebuild.sdp.md:103 requires \"the application's tenant list\" without declaring its authority or enumeration, so an all-tenant pass cannot locate its next tenant. The issue and structured enumeration or shared compare-helper fix remain right; completion must require every required check passed. The build must show: convex-test and native backend enumeration must visit every required tenant and exercise a tenant created behind the cursor under the ruled online or paused semantics, without treating absent tenant scope as a wildcard. S4 on paper, 2026-10-02. `spec:command.actor-and-scope` pins the `tenants` table, `insertGrant` writing a tenant's row and `nextTenant` (E-49); `spec:application.rebuild` `step2`, `step6` and the rule on a tenant behind the cursor iterate it, and `spec:application.restore` `validatorCheckScope` points at `nextTenant`. The receipt sweep of `spec:command.receipt-table` does not name it yet, for the unit that builds the sweep. The build must show at convex-test and on the native backend: two tenants and the cursor crossing, an empty tenant, and a tenant inserted behind the cursor. The online rebuild of the order summary: The tenant list is the parent's `tenants` table, filled from the grants by `fillTenants` (rebuild.sdp.md:84, fillSteps), and the backfill crosses tenants in tenant-ID order; an empty tenant and a tenant behind the cursor run at `convex-test`, two tenants natively in L2-5. Partially fixed: the native empty tenant and tenant behind the cursor, and the receipts sweep's tenant iteration.", "slice": "S4", "detector": "judgement" }, @@ -2257,8 +2257,8 @@ ], "issue": "A verify batch 'checks that every stream whose DTO projects to a row has one', 'while a stream whose DTO projects to `null` needs no row'. On a first build that is enough, because a live delete removes a row that exists. After `rollbackGeneration` the retired generation still holds the row of a subject deleted between the switch and the rollback, the verify pass does not look at it, and the reopened generation becomes active showing a subject that no longer projects to a row.", "fix": "Have verify call `applyProjection` in backfill mode for a DTO that projects to `null` as well, which deletes the row if present and counts it as a repaired miss, and say so in the verify flow and in the L2-5 example's rollback step.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-06: At HEAD, rebuild.sdp.md:81 says a DTO projecting to null \"needs no row\", and :108 reopens the retired generation for verification, so rollback can retain and disclose the retired row of a deleted subject. Keep the absence-repair fix and raise minor to major for returned-data correctness. The build must show: convex-test and native backend rollback must leave a subject deleted during retirement absent from the restored generation and its list subscription after switch, with source enumeration including deleted subjects. S4 on paper, 2026-10-02. `spec:application.rebuild` `verifySteps` and `spec:application.projection-contract` `step1` and the backfill-mode rule delete the row of a DTO that projects to `null`, so one verify pass repairs absence as well as presence. The build must show at convex-test a subject deleted between the switch and the rollback absent from the reopened generation after its verify pass; the native step of `spec:application.rebuild.online-rebuild-interrupt-resume` allows a cancellation in place of a deletion, so the native share of absence is still to be shown.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-06: At HEAD, rebuild.sdp.md:81 says a DTO projecting to null \"needs no row\", and :108 reopens the retired generation for verification, so rollback can retain and disclose the retired row of a deleted subject. Keep the absence-repair fix and raise minor to major for returned-data correctness. The build must show: convex-test and native backend rollback must leave a subject deleted during retirement absent from the restored generation and its list subscription after switch, with source enumeration including deleted subjects. S4 on paper, 2026-10-02. `spec:application.rebuild` `verifySteps` and `spec:application.projection-contract` `step1` and the backfill-mode rule delete the row of a DTO that projects to `null`, so one verify pass repairs absence as well as presence. The build must show at convex-test a subject deleted between the switch and the rollback absent from the reopened generation after its verify pass; the native step of `spec:application.rebuild.online-rebuild-interrupt-resume` allows a cancellation in place of a deletion, so the native share of absence is still to be shown. The online rebuild of the order summary: A `null` candidate deletes the row in every mode, and the rollback's verify pass removes the row of a deleted subject, proven at `convex-test` on the fixture's `documentTitle` version 2. Partially fixed: the native absence after a rollback.", "slice": "S4", "detector": "convex-test" }, @@ -3910,7 +3910,7 @@ "issue": "Rounds 1 and 2 moved the adapter, the list page, the backfill, the baseline batch and the restore's stream check from document counts to bytes. Six batches still state a bound in documents only, and at the corpus's own row ceilings each crosses 'Data read 16 MiB' or 'Data written 16 MiB' per transaction (docs.convex.dev/production/state/limits#transactions, read 2026-10-01), against rubric item 29. (1) `receipts.sweep`: 'at most 1,000 receipts per run for one tenant ... so one run stays far under the 16,000 documents written per transaction' (idempotency `limitSweepBatch`, receipt-table `fnSweep`). receipt-table `limitRowSize` lets a row reach 400 KiB; the receipt of the experiment's own 100-line `PlaceOrder` holds 101 affected refs and 101 stream versions, about 30 KiB, so 1,000 such rows read about 29 MiB. A mutation is deterministic, so a sweep that crosses the ceiling fails at the same rows on every run and expired receipts are never removed. (2) restore `fnReceiptCheckBatch`: '25 receipts per receipt batch, each one component call', where the call is `byOperation`, which 'ranges `by_operation` and collects' every event of the operation; 25 receipts at 400 KiB are 9.8 MiB before any event, and one operation may have appended up to 8 MiB of events (adapter `limitDocumentsWrittenPerCall`), so the batch has no byte bound at all. A failed check batch leaves the restore closed. (3) rebuild `limitPurgeBatch`: '500 rows deleted per purge batch, one index range, under 1 second'; at the 64 KiB `rowBudgetBytes` ceiling 500 rows read 31 MiB, while the backfill beside it is derived from the same budget. (4) sweeper `limitSweepBatch`: '100 candidates per scan and 100 expired leases per scan as the document bound, so one run reads at most about 400 documents', with an inline payload of up to 64 KiB on each row (record-contract `limitInlinePayload`) and each candidate read again inside `recoverOne`, up to about 19 MiB under the shared limits. (5) retention `limitRetentionBatch`: '500 deletions and 500 compactions per run ... under the 16,000 writes per transaction', up to 32 MiB read for the deletions alone. (6) fan-out `limitFanOutPage`: '500 derived obligations per fan-out attempt ... under the 1000 schedules per mutation and the write ceilings', which at a 64 KiB payload writes 31 MiB. The first three are Layer 1 and 2 and sit in the first experiment's path.", "fix": "Derive sweep, receipt-check, purge, retention and fan-out bounds from complete measured row and event costs, including rereads, metadata and schedules. Page operation history with a resumable byte and row bound and persist caller progress across pages. At 64 KiB, floor(4 MiB / 64 KiB) is 64, not 62, and payload size alone is not row cost. If transaction metrics select headroom, await the pinned API and read metrics.bytesRead.remaining from the resolved result; each metric has { used, remaining } fields. Preserve provisional numeric defaults until native backend boundary cases prove progress within shared limits.", "status": "partially-fixed", - "note": "the platform review of 2026-10-02, APPL-08: At HEAD, rebuild.sdp.md:124 deletes \"500 rows\" and restore.sdp.md:100 checks \"25 receipts\" with unbounded byOperation calls, so 64 KiB purge rows alone read 31.25 MiB and one migration operation can exceed a transaction. The issue holds, but STATE.md Leads 2 invalidates the fix's 62-row arithmetic, payload-as-row estimate and synchronous bytesRead access; derive complete row costs, page operation history and use any metric through its awaited { used, remaining } shape. the platform review of 2026-10-02, CORE-07: At HEAD, context/queries.sdp.md:81 says byOperation \"collects\" while journal.sdp.md:106 gives every baseline in a migration one operationId, so 100 baselines near 256 KiB exceed 25 MiB under one ID despite safe individual batches. Bound operation-history pages and the receipt-check caller by bytes and rows; a resumable answer must let the next batch advance rather than retry the same refused operation. The build must show: Native backend cases must purge near-limit read-model rows across tenants and check receipts for a multi-batch migration whose total baseline payload exceeds one transaction, with bounded operation-history pages reaching every event once. Sweep, retention and fan-out boundary cases must advance the exact saved position without repeated limit failures, counting complete row cost, rereads and schedules. S4 on paper, 2026-10-02. `spec:application.rebuild` `limitPurgeBatch` derives the purge count from `rowBudgetBytes` and states its read with each row's system fields, and `purgeSteps` iterates tenants. The native backend must show a purge near its bound; receipts, restore, obligations, retention and fan-out stay.", + "note": "the platform review of 2026-10-02, APPL-08: At HEAD, rebuild.sdp.md:124 deletes \"500 rows\" and restore.sdp.md:100 checks \"25 receipts\" with unbounded byOperation calls, so 64 KiB purge rows alone read 31.25 MiB and one migration operation can exceed a transaction. The issue holds, but STATE.md Leads 2 invalidates the fix's 62-row arithmetic, payload-as-row estimate and synchronous bytesRead access; derive complete row costs, page operation history and use any metric through its awaited { used, remaining } shape. the platform review of 2026-10-02, CORE-07: At HEAD, context/queries.sdp.md:81 says byOperation \"collects\" while journal.sdp.md:106 gives every baseline in a migration one operationId, so 100 baselines near 256 KiB exceed 25 MiB under one ID despite safe individual batches. Bound operation-history pages and the receipt-check caller by bytes and rows; a resumable answer must let the next batch advance rather than retry the same refused operation. The build must show: Native backend cases must purge near-limit read-model rows across tenants and check receipts for a multi-batch migration whose total baseline payload exceeds one transaction, with bounded operation-history pages reaching every event once. Sweep, retention and fan-out boundary cases must advance the exact saved position without repeated limit failures, counting complete row cost, rereads and schedules. S4 on paper, 2026-10-02. `spec:application.rebuild` `limitPurgeBatch` derives the purge count from `rowBudgetBytes` and states its read with each row's system fields, and `purgeSteps` iterates tenants. The native backend must show a purge near its bound; receipts, restore, obligations, retention and fan-out stay. The online rebuild of the order summary: The purge batch is now bounded by `limitPurgeBatch(readModel)` from the row budget (rebuild.sdp.md:147), proven at `convex-test`. Partially fixed: the native purge near its bound, the receipts sweep and the restore's checks.", "slice": "S4", "detector": "native local-backend test" }, @@ -3942,8 +3942,8 @@ ], "issue": "During an online rebuild every live command and every backfill batch meet on one document. The pipeline's step 9 calls `generationsToWrite`, which makes 'at most two indexed reads' of `by_read_model_state` and so reads the row of the generation in `building`, `verifying` or `verified`. The same row is the batch's checkpoint: 'The generation row carries the backfill cursor, the verify cursor, the fence and the counts, and a batch writes them in the same mutation as its rows' (generation-registry Contract), and `step5` 'patches the generation row with the new cursor, the counts and `updatedAt`' on every batch, with batches chained by `ctx.scheduler.runAfter(0, ...)`. Convex decides conflicts on the read set: 'At the end of the transaction, the writes all commit if every version in the read set is still the latest version of that record' (docs.convex.dev/database/advanced/occ, read 2026-10-01), and a write to a document inside a range a mutation read conflicts whatever field changed ('any change to the \"tasks\" table will conflict with the `writeCount` mutation', docs.convex.dev/error#1, read 2026-10-01). So each batch commit invalidates every command on that read model that is in flight, on any tenant, and the command re-runs. A caller's mutation gets a bounded number of retries, after which 'some of the invocations will eventually throw this error' (same page). With batches committing back to back, a short command re-runs now and then and a long one, the 100-line `PlaceOrder`, overlaps a batch commit on most tries and can exhaust its retries, for the whole length of the rebuild. The corpus says the opposite: 'The common rebuild costs no downtime' (rebuild value), 'The registry is data that changes only during a rebuild, so it is not an application-wide counter in the write path' (generation-registry), and it records the one-time re-run for the gate ('a gate change makes every concurrent command re-run once') but not this repeated one. Probe 6 races a batch against a command on a read-model row, not on the registry row. How often a command loses is unmeasured; that it conflicts on every overlapping batch commit is documented behavior.", "fix": "In `spec:application.generation-registry` split the row: keep on `generations` only what a command and a query read, `readModel`, `generation`, `projectionVersion`, `state`, `pauseRequired` and `fence`, which change at start, switch, resume, rollback and abort; move `cursor`, `verifyCursor`, `batchSize`, `batchesDone`, `rowsWritten`, `rowsSkipped`, `misses`, `lastError` and `updatedAt` to a `generationProgress` table keyed by generation ID that only batches and operators read. A batch then checks state and fence on the registry row, which it reads and does not write, and checkpoints on the progress row, which no command reads. Update rebuild's steps 1 and 5, `resumeChain`, the restore's inventory and `finishRestoreRun` to the two rows. Add the registry-row case to Probe 6 in `spec:facts.probe-plan` and to the first experiment's contention run: `PlaceOrder` at the maximum while a backfill runs, counting retries and exhaustions.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-07: At HEAD, generation-registry.sdp.md:47 makes the registry row \"the checkpoint\" and rebuild.sdp.md:115 patches it every batch, so each overlapping command selecting that row can conflict even on unrelated streams. The issue and separate progress-row fix remain right; the frequency and throughput consequence need native backend measurement, not convex-test. The build must show: Source review must show disjoint command-selection and batch-checkpoint write sets, and a native backend comparison must run maximum-size and unrelated-stream commands during backfill against a no-rebuild control, reporting retries and exhaustions rather than inferring throughput from convex-test. S4 on paper, 2026-10-02. `spec:application.generation-registry` `tableGenerationProgress`, `transitionCost` and the checkpoint question under E-40 move the cursor, the batch size and the counts to a progress row no command or query reads, and a batch patches the generation row only in the mutation that ends its pass. The build must show, in a contention run on the native backend against a run with no rebuild, that no command fails on a registry conflict while batches checkpoint.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-07: At HEAD, generation-registry.sdp.md:47 makes the registry row \"the checkpoint\" and rebuild.sdp.md:115 patches it every batch, so each overlapping command selecting that row can conflict even on unrelated streams. The issue and separate progress-row fix remain right; the frequency and throughput consequence need native backend measurement, not convex-test. The build must show: Source review must show disjoint command-selection and batch-checkpoint write sets, and a native backend comparison must run maximum-size and unrelated-stream commands during backfill against a no-rebuild control, reporting retries and exhaustions rather than inferring throughput from convex-test. S4 on paper, 2026-10-02. `spec:application.generation-registry` `tableGenerationProgress`, `transitionCost` and the checkpoint question under E-40 move the cursor, the batch size and the counts to a progress row no command or query reads, and a batch patches the generation row only in the mutation that ends its pass. The build must show, in a contention run on the native backend against a run with no rebuild, that no command fails on a registry conflict while batches checkpoint. The online rebuild of the order summary: The rebuild's checkpoint moved to the progress row (generation-registry.sdp.md, tableGenerationProgress; rebuild.sdp.md:107), and the L2-5 native run asserts that no PlaceOrder, CancelOrder or parent query conflicts on it and that the reruns naming `generations` stay within the bound of rebuild.online-rebuild-interrupt-resume.sdp.md:36. Partially fixed: the comparison with a run with no rebuild that the issue asks for was declined, and an interrupt that meets a batch may rerun on the progress row, counted beside the generations reruns.", "slice": "S4", "detector": "measurement or contention run" }, From 8a6af06cc0103a243e87299539026b89baddb049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 05:24:34 +0200 Subject: [PATCH 101/155] build: 3b fold-in the moved cursor kept pageInGeneration takes the index's field count and drops _creationTime and _id only from a key that carries them, so a cursor already moved, which paginator returns as a pinned page's continueCursor, keeps its length and has only its generation replaced. The Spec's signature and the list's call name the count; a convex-test case pages on from a pinned page across a switch, and the online rebuild pages on from its first subscription's page after each switch. Co-Authored-By: Claude Opus 5.5 --- design/specs/application/read-models.sdp.md | 4 +-- example/convex/readModels.ts | 6 +++- fixture/convex/readModels.ts | 6 +++- src/read-model/queries.ts | 26 ++++++++++++----- ...ld.online-rebuild-interrupt-resume.test.ts | 9 ++++++ .../read-model-page-in-generation.test.ts | 28 +++++++++++++++++-- 6 files changed, 65 insertions(+), 14 deletions(-) diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index 48dfb7b..b6685f8 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -93,8 +93,8 @@ Nothing here schedules, and nothing here reads `ctx.auth` inside a component. Th - transactionBoundary: read-model writes happen inside the command's top-level mutation, after the context calls; reads are parent queries, reactive by subscription (D8, D1, F7) - convexSurface: one static `query` export per read need in the parent; component queries are reached through `ctx.runQuery(components.orders.queries.order.get, args)`, its siblings `list`, `history` and `rebuild`, and `components.orders.queries.operations.byOperation`, as the queries contract names them; no scheduled function, no cron, no action (D8, F11) - fnEntityDetail: `getOrder` as `spec:context.queries` pins it under `parentGet`: it calls `authorizeQuery` and then `ctx.runQuery(components.orders.queries.order.get, { tenantId, streamId: orderId })` (D8, D11, F15) -- fnListOrSummary: `export const listOrderSummaries = query({ args: { tenantId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(orderSummaryView), handler: async (ctx, { tenantId, status, paginationOpts }) => { await authorizeQuery(ctx, { name: "listOrderSummaries", tenantId, permission: "orders.read" }); const generation = await activeGeneration(ctx, "orderSummary"); if (generation === undefined) throw new Error("The read model orderSummary has no active generation"); const result = await paginator(ctx.db, schema).query("orderSummaries").withIndex("by_status", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("status", status)).paginate(boundedPage(pageInGeneration(paginationOpts, generation), limitReadModelList(orderSummary))); return { ...result, page: result.page.map(readModelView) }; } })` one `paginate` of `paginator` from `convex-helpers/server/pagination` with the composition's `schema`, over the active generation's rows in `placedAt` then `key` order within a status, where `orderSummary` is the read model, whose name is `"orderSummary"` and whose table is `orderSummaries` (D8, D9, Law 5, Law 11, E-24) -- fnPageInGeneration: `pageInGeneration(opts: PaginationOptions, generation: number): PaginationOptions` returns `opts` with its `cursor` and its `endCursor`, when set, each moved into `generation`: a cursor of `paginator` is the JSON of a row's index key, so the function replaces the key's second element, the generation that wrote the row, with `generation` and drops the last two, `_creationTime` and `_id`, which belong to that generation's document; a cursor that is not such a key, `null` or `undefined` is returned as it is, and the read-model library exports the function beside `readModelView` (D9, E-24, Probe 5) +- fnListOrSummary: `export const listOrderSummaries = query({ args: { tenantId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(orderSummaryView), handler: async (ctx, { tenantId, status, paginationOpts }) => { await authorizeQuery(ctx, { name: "listOrderSummaries", tenantId, permission: "orders.read" }); const generation = await activeGeneration(ctx, "orderSummary"); if (generation === undefined) throw new Error("The read model orderSummary has no active generation"); const result = await paginator(ctx.db, schema).query("orderSummaries").withIndex("by_status", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("status", status)).paginate(boundedPage(pageInGeneration(paginationOpts, generation, 5), limitReadModelList(orderSummary))); return { ...result, page: result.page.map(readModelView) }; } })` one `paginate` of `paginator` from `convex-helpers/server/pagination` with the composition's `schema`, over the active generation's rows in `placedAt` then `key` order within a status, where `orderSummary` is the read model, whose name is `"orderSummary"` and whose table is `orderSummaries` (D8, D9, Law 5, Law 11, E-24) +- fnPageInGeneration: `pageInGeneration(opts: PaginationOptions, generation: number, indexFields: number): PaginationOptions` returns `opts` with its `cursor` and its `endCursor`, when set, each moved into `generation`: a cursor of `paginator` is the JSON of a row's index key, so the function replaces the key's second element, the generation that wrote the row, with `generation` and drops the last two, `_creationTime` and `_id`, which belong to that generation's document, when the key carries them, which it tells by a length of `indexFields`, the index's own field count, plus two, so that a cursor already moved, which `paginator` returns as a pinned page's `continueCursor`, keeps its length and has only its generation replaced; a cursor that is not such a key, `null` or `undefined` is returned as it is, and the read-model library exports the function beside `readModelView` (D9, E-24, Probe 5) - indexReadModelList: the index a read-model list reads has `tenantId`, `generation` and the list's equality fields first, then its order field, then `key`, as `by_status` of `tableOrderSummaries` in `spec:application.projection-contract`, so a position within a tenant names at most one row of each generation and a cursor moved into another generation lands between the same rows there (D9, E-9, Law 11) - fnReadModelView: `readModelView(row: Row): Omit` what a read-model query returns for a row: the tenant, the key, the projection version, the source versions and the projected fields; the built-in page's rows carry `_id` and `_creationTime`, which an object validator of the row's fields refuses, and no document ID or generation number leaves the parent (D8, D2, E-9) - validatorReadModelView: `orderSummaryView = v.object({ tenantId: v.string(), key: v.string(), projectionVersion: v.number(), sourceVersions: v.array(streamVersionValidator), ...orderSummaryFields })` where `orderSummaryFields` are the projected fields `spec:application.projection-contract` declares for `tableOrderSummaries` (E-9, D8) diff --git a/example/convex/readModels.ts b/example/convex/readModels.ts index 3f04a73..4777588 100644 --- a/example/convex/readModels.ts +++ b/example/convex/readModels.ts @@ -32,6 +32,10 @@ const orderSummaryView = v.object({ sourceVersions: v.array(streamVersionValidator), ...orderSummaryFields, }); +// The fields of the list's index, which tell a cursor that carries a document's system fields. +const byStatusFields = schema.tables.orderSummaries[" indexes"]().find( + (index) => index.indexDescriptor === "by_status", +)!.fields.length; // A tenant's order summaries of one status, in the order they were placed. export const listOrderSummaries = query({ args: { @@ -61,7 +65,7 @@ export const listOrderSummaries = query({ ) .paginate( boundedPage( - pageInGeneration(paginationOpts, generation), + pageInGeneration(paginationOpts, generation, byStatusFields), limitReadModelList(orderSummary), ), ); diff --git a/fixture/convex/readModels.ts b/fixture/convex/readModels.ts index 44fb185..9eb885b 100644 --- a/fixture/convex/readModels.ts +++ b/fixture/convex/readModels.ts @@ -30,6 +30,10 @@ const documentSummaryView = v.object({ sourceVersions: v.array(streamVersionValidator), ...documentSummaryFields, }); +// The fields of the list's index, which tell a cursor that carries a document's system fields. +const byStatusFields = schema.tables.documentSummaries[" indexes"]().find( + (index) => index.indexDescriptor === "by_status", +)!.fields.length; // A tenant's document summaries of one status, in document ID order. export const listDocumentSummaries = query({ args: { @@ -59,7 +63,7 @@ export const listDocumentSummaries = query({ ) .paginate( boundedPage( - pageInGeneration(paginationOpts, generation), + pageInGeneration(paginationOpts, generation, byStatusFields), limitReadModelList(documentSummary), ), ); diff --git a/src/read-model/queries.ts b/src/read-model/queries.ts index 422d025..710d61f 100644 --- a/src/read-model/queries.ts +++ b/src/read-model/queries.ts @@ -23,18 +23,26 @@ export const limitReadModelList = (readModel: AnyReadModel): PageLimit => ({ }); // A client's page options with each cursor moved into the generation now active. A cursor of // paginator is the JSON of a row's index key, whose second element is the generation that wrote the -// row and whose last two are that document's _creationTime and _id; the moved cursor names the same -// position by the order field and key alone. +// row; a key of the index's fields plus two also carries that document's _creationTime and _id, which +// are dropped. A cursor already moved keeps its length, so moving it again replaces the generation alone. export function pageInGeneration( opts: PaginationOptions, generation: number, + indexFields: number, ): PaginationOptions { - const moved = { ...opts, cursor: moveCursor(opts.cursor, generation) }; + const moved = { + ...opts, + cursor: moveCursor(opts.cursor, generation, indexFields), + }; if (opts.endCursor !== undefined && opts.endCursor !== null) - moved.endCursor = moveCursor(opts.endCursor, generation); + moved.endCursor = moveCursor(opts.endCursor, generation, indexFields); return moved; } -function moveCursor(cursor: string | null, generation: number) { +function moveCursor( + cursor: string | null, + generation: number, + indexFields: number, +) { if (cursor === null) return cursor; let key: unknown; try { @@ -42,7 +50,11 @@ function moveCursor(cursor: string | null, generation: number) { } catch { return cursor; } - if (!Array.isArray(key) || key.length < 4 || typeof key[1] !== "number") + if ( + !Array.isArray(key) || + (key.length !== indexFields && key.length !== indexFields + 2) || + typeof key[1] !== "number" + ) return cursor; - return JSON.stringify([key[0], generation, ...key.slice(2, -2)]); + return JSON.stringify([key[0], generation, ...key.slice(2, indexFields)]); } diff --git a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts index 47459cf..f89c50c 100644 --- a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts +++ b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts @@ -490,6 +490,15 @@ async function switchAndObserve(w: World, generationId: string) { // rebuild.online-rebuild-interrupt-resume.sdp.md:37: each subscription retains its rows across the switch. expect(value.page).toEqual(before[i]); } + // read-models.sdp.md fnPageInGeneration: the first page's cursor pages on in the generation now active. + const first = w.watches[0]!.values.at(-1)!; + const next = await w.clients[0]!.query(api.readModels.listOrderSummaries, { + tenantId: tenants[0], + status: "placed", + paginationOpts: { cursor: first.continueCursor, numItems: 100 }, + }); + expect(next.page.map(({ key }) => key)).not.toContain(first.page.at(-1)?.key); + expect(next.page.length).toBeGreaterThan(0); await drain(w); } diff --git a/tests/simulator/read-model-page-in-generation.test.ts b/tests/simulator/read-model-page-in-generation.test.ts index 4a7621d..e439792 100644 --- a/tests/simulator/read-model-page-in-generation.test.ts +++ b/tests/simulator/read-model-page-in-generation.test.ts @@ -16,6 +16,8 @@ const orders = [ ["d", 3], ["e", 4], ] as const; +// tenantId, generation, status, placedAt and key: by_status of orderSummaries. +const byStatusFields = 5; type Page = { rows: string[]; isDone: boolean; continueCursor: string }; async function seeded() { @@ -53,7 +55,7 @@ function page( .eq("generation", generation) .eq("status", "placed"), ) - .paginate(pageInGeneration(opts, generation)); + .paginate(pageInGeneration(opts, generation, byStatusFields)); return { rows: result.page.map((row) => `${row.key}@${row.generation}`), isDone: result.isDone, @@ -84,6 +86,20 @@ test("convex-test: a pinned page shows the switched-to generation's rows up to i expect(pinned.isDone).toBe(false); }); +test("convex-test: a pinned page's continueCursor, already moved, pages on across a switch", async () => { + const t = await seeded(); + const first = await page(t, 1, { cursor: null, numItems: 2 }); + const pinned = await page(t, 2, { + cursor: null, + endCursor: first.continueCursor, + numItems: 2, + }); + expect(pinned.rows).toEqual(["a@2", "b@2"]); + expect( + (await page(t, 1, { cursor: pinned.continueCursor, numItems: 2 })).rows, + ).toEqual(["c@1", "d@1"]); +}); + test("convex-test: after a switch back a cursor of generation 2 continues from the same position in generation 1", async () => { const t = await seeded(); const first = await page(t, 2, { cursor: null, numItems: 2 }); @@ -106,11 +122,17 @@ test("convex-test: a page edge on an order-field tie neither repeats nor skips a test("convex-test: pageInGeneration returns a cursor that is not an index key as it is", () => { for (const cursor of [null, "[]", "not json", '{"a":1}', '["a","b","c","d"]']) - expect(pageInGeneration({ cursor, numItems: 1 }, 2)).toStrictEqual({ + expect( + pageInGeneration({ cursor, numItems: 1 }, 2, byStatusFields), + ).toStrictEqual({ cursor, numItems: 1, }); expect( - pageInGeneration({ cursor: null, endCursor: null, numItems: 1 }, 2), + pageInGeneration( + { cursor: null, endCursor: null, numItems: 1 }, + 2, + byStatusFields, + ), ).toStrictEqual({ cursor: null, endCursor: null, numItems: 1 }); }); From 314c265f7905ce84bf2bd701e92a039cf8bd3ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 05:25:48 +0200 Subject: [PATCH 102/155] design: 3b four register rows decided by their advisors' checked leans OD-057, OD-071, OD-075 and OD-080 are decided from the leans sol checked in the tooling unit, applied by the tool as the fold-in of 3b builds their text; OD-149 stays sorted because its lean was not checked. The owner's words: "I agree with all recommendations". Co-Authored-By: Claude Fable 5.1 --- design/decisions/register.json | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/design/decisions/register.json b/design/decisions/register.json index 1cf4705..07a3bdd 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -2229,7 +2229,7 @@ "provisionalReading": null, "blocks": "S4", "advisor": "convex", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "delegated", "reason": "Rule 3: which Convex mechanism drives a backfill batch, settled by a probe and taken back by an edit [design/specs/facts/f17-migrations-fits-generation-backfill.sdp.md:22; design/README.md:219].", @@ -2243,9 +2243,9 @@ }, "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held", "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "advisor:convex", + "decidedOn": "2026-10-03", + "ruling": "By the advisor's checked lean, which the owner agreed to (\"I agree with all recommendations\", rulings 1 of round 3, Q8): the lean as the rebuild Specs build it, every batch its own self-scheduled internal mutation, @convex-dev/migrations used for nothing, the progress row the one checkpoint" }, { "id": "OD-058", @@ -2819,7 +2819,7 @@ "provisionalReading": null, "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "delegated", "reason": "Rule 3, the shape of a table and which row holds a cursor; the doc binds batching to the component and fixes no checkpoint [design/specs/application/generation-registry.sdp.md:37]; an edit to one Spec and one file takes it back. Doubt: the lean goes against the built reading of [design/specs/application/generation-registry.sdp.md:47] on a contention judgment that only the native tier shows, and S4's guidance [design/STATE.md:160], a pointer, asks for progress away from rows every command reads.", @@ -2833,9 +2833,9 @@ }, "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held", "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "advisor:domain", + "decidedOn": "2026-10-03", + "ruling": "By the advisor's checked lean, which the owner agreed to (\"I agree with all recommendations\", rulings 1 of round 3, Q8): the lean as built, the generation row patched by a batch only at a pass end and the progress row the checkpoint; an interrupt that meets a batch may rerun on the progress row, counted beside the generations reruns" }, { "id": "OD-072", @@ -2990,7 +2990,7 @@ "provisionalReading": "The option taken here: a per-entity read model declared as a `ReadModel` with `name`, `table`, `rowBudgetBytes` and its `Projection`, which holds `version`, `keyOf` and `project` alone, rows with `tenantId`, `generation`, `key`, `projectionVersion` and `sourceVersions`, one `applyProjection` helper with a mode per caller that writes each generation by its role, active or building, for an `AggregateProjection` an aggregate row with `value` and `entityCount` and no `sourceVersions`, because the markers carry them, written by `applyAggregate` under the marker's version compare with the mode ignored, and for a history-dependent view a `HistoryProjection` that folds events into the prior row, fed on the live path by the events the operation appended, which the adapter's `streams` entry now carries, and on the backfill by the context's `history` pages under the write pause, with `sourceVersions` recording the highest version folded per stream so that an event is never folded twice; a cross-context or cross-stream row is built by one projection bound once per source stream type; the owner confirms the three forms and has not ruled on the split of a per-entity read model into `ReadModel` and `Projection`, on the write by generation role or on the row measured against its budget at every write; no check ties a generation to the projection version it records, and how a command writes two projection versions at once is the next question (E-9, E-23, E-24, D8, D9)", "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "delegated", "reason": "Rule 3, signatures and the shape of a row: the doc gives no signature, row shape or input for the three forms [design/specs/application/projection-contract.sdp.md:33]; an edit to one Spec and the read-model files takes it back. The order in which the forms are bound is the main thread's [design/STATE.md:160], a pointer.", @@ -3004,9 +3004,9 @@ }, "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held", "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "advisor:domain", + "decidedOn": "2026-10-03", + "ruling": "By the advisor's checked lean, which the owner agreed to (\"I agree with all recommendations\", rulings 1 of round 3, Q8): the lean as built, a ReadModel with its projections, the latest first, and applyProjection writing each generation by its role and its own projection version" }, { "id": "OD-076", @@ -3220,7 +3220,7 @@ "provisionalReading": "The option taken here: a batch is one mutation with one cursor and a bound of at most 100 entities, capped twice, by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget, a design choice under F13 that is a field of the generation row and that `resumeGeneration` lowers down to 1 when a generation stalls on optimistic-concurrency conflicts over a hot key range, because a mutation that loses the race never commits and cannot count its own losses; the migrations component drives batches whose source is a parent table, and a self-scheduled internal mutation with the same batch contract drives batches whose source is a context's enumeration until Probe 6 shows the component can; Probe 6 also measures the contention and decides whether an automatic stall check earns its function (E-40, E-24, E-9, F17, F13, F1)", "blocks": "S4", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "delegated", "reason": "Rule 3, a bound and its number and which mechanism drives a batch: the doc gives no batch size [design/specs/application/rebuild.sdp.md:58], and the Spec itself leaves the component's reach to Probe 6, so a probe comes first.", @@ -3234,9 +3234,9 @@ }, "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held", "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "advisor:domain", + "decidedOn": "2026-10-03", + "ruling": "By the advisor's checked lean, which the owner agreed to (\"I agree with all recommendations\", rulings 1 of round 3, Q8): the lean as built, the batch size the read model's by batchSizeFor and a field of the progress row, the context's list page bounding a batch's bytes and not its size" }, { "id": "OD-081", From ade199d6bc097c0c49f9ec3f066f84109a7814d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 05:48:23 +0200 Subject: [PATCH 103/155] build: 3b fold-in a cursor from another list is refused pageInGeneration takes the query's equality prefix and refuses, before the list reads a row, a cursor whose tenant or status differs, whose generation is not a number or whose key is neither the index's length nor that plus two, with "The cursor is not from this list". The page-in-generation cases run through both compositions' list entries. Co-Authored-By: Claude Opus 5.5 --- design/specs/application/read-models.sdp.md | 4 +- example/convex/readModels.ts | 6 +- fixture/convex/readModels.ts | 6 +- src/read-model/queries.ts | 32 ++- .../read-model-page-in-generation.test.ts | 250 +++++++++++++----- 5 files changed, 212 insertions(+), 86 deletions(-) diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index b6685f8..63b559c 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -93,8 +93,8 @@ Nothing here schedules, and nothing here reads `ctx.auth` inside a component. Th - transactionBoundary: read-model writes happen inside the command's top-level mutation, after the context calls; reads are parent queries, reactive by subscription (D8, D1, F7) - convexSurface: one static `query` export per read need in the parent; component queries are reached through `ctx.runQuery(components.orders.queries.order.get, args)`, its siblings `list`, `history` and `rebuild`, and `components.orders.queries.operations.byOperation`, as the queries contract names them; no scheduled function, no cron, no action (D8, F11) - fnEntityDetail: `getOrder` as `spec:context.queries` pins it under `parentGet`: it calls `authorizeQuery` and then `ctx.runQuery(components.orders.queries.order.get, { tenantId, streamId: orderId })` (D8, D11, F15) -- fnListOrSummary: `export const listOrderSummaries = query({ args: { tenantId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(orderSummaryView), handler: async (ctx, { tenantId, status, paginationOpts }) => { await authorizeQuery(ctx, { name: "listOrderSummaries", tenantId, permission: "orders.read" }); const generation = await activeGeneration(ctx, "orderSummary"); if (generation === undefined) throw new Error("The read model orderSummary has no active generation"); const result = await paginator(ctx.db, schema).query("orderSummaries").withIndex("by_status", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("status", status)).paginate(boundedPage(pageInGeneration(paginationOpts, generation, 5), limitReadModelList(orderSummary))); return { ...result, page: result.page.map(readModelView) }; } })` one `paginate` of `paginator` from `convex-helpers/server/pagination` with the composition's `schema`, over the active generation's rows in `placedAt` then `key` order within a status, where `orderSummary` is the read model, whose name is `"orderSummary"` and whose table is `orderSummaries` (D8, D9, Law 5, Law 11, E-24) -- fnPageInGeneration: `pageInGeneration(opts: PaginationOptions, generation: number, indexFields: number): PaginationOptions` returns `opts` with its `cursor` and its `endCursor`, when set, each moved into `generation`: a cursor of `paginator` is the JSON of a row's index key, so the function replaces the key's second element, the generation that wrote the row, with `generation` and drops the last two, `_creationTime` and `_id`, which belong to that generation's document, when the key carries them, which it tells by a length of `indexFields`, the index's own field count, plus two, so that a cursor already moved, which `paginator` returns as a pinned page's `continueCursor`, keeps its length and has only its generation replaced; a cursor that is not such a key, `null` or `undefined` is returned as it is, and the read-model library exports the function beside `readModelView` (D9, E-24, Probe 5) +- fnListOrSummary: `export const listOrderSummaries = query({ args: { tenantId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(orderSummaryView), handler: async (ctx, { tenantId, status, paginationOpts }) => { await authorizeQuery(ctx, { name: "listOrderSummaries", tenantId, permission: "orders.read" }); const generation = await activeGeneration(ctx, "orderSummary"); if (generation === undefined) throw new Error("The read model orderSummary has no active generation"); const result = await paginator(ctx.db, schema).query("orderSummaries").withIndex("by_status", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("status", status)).paginate(boundedPage(pageInGeneration(paginationOpts, [tenantId, generation, status], 5), limitReadModelList(orderSummary))); return { ...result, page: result.page.map(readModelView) }; } })` one `paginate` of `paginator` from `convex-helpers/server/pagination` with the composition's `schema`, over the active generation's rows in `placedAt` then `key` order within a status, where `orderSummary` is the read model, whose name is `"orderSummary"` and whose table is `orderSummaries` (D8, D9, Law 5, Law 11, E-24) +- fnPageInGeneration: `pageInGeneration(opts: PaginationOptions, prefix: Value[], indexFields: number): PaginationOptions` returns `opts` with its `cursor` and its `endCursor`, when set, each moved into the generation of `prefix`, the query's equality values in index order, `tenantId`, `generation` and the list's equality fields, the generation second: a cursor of `paginator` is the JSON of a row's index key, so the function replaces the key's second element, the generation that wrote the row, with the prefix's and drops the last two, `_creationTime` and `_id`, which belong to that generation's document, when the key carries them, which it tells by a length of `indexFields`, the index's own field count, plus two, so that a cursor already moved, which `paginator` returns as a pinned page's `continueCursor`, keeps its length and has only its generation replaced; a cursor whose key's length is neither `indexFields` nor `indexFields` plus two, whose generation is not a number, whose other equality elements differ from the prefix's, or that is not a key, throws a plain `Error` `"The cursor is not from this list"` before the list reads a row, so a saved cursor never overrides the query's tenant or status; `null`, `undefined` and `"[]"`, the helper's end of the list, are returned as they are, and the read-model library exports the function beside `readModelView` (D9, E-24, Probe 5, Law 11) - indexReadModelList: the index a read-model list reads has `tenantId`, `generation` and the list's equality fields first, then its order field, then `key`, as `by_status` of `tableOrderSummaries` in `spec:application.projection-contract`, so a position within a tenant names at most one row of each generation and a cursor moved into another generation lands between the same rows there (D9, E-9, Law 11) - fnReadModelView: `readModelView(row: Row): Omit` what a read-model query returns for a row: the tenant, the key, the projection version, the source versions and the projected fields; the built-in page's rows carry `_id` and `_creationTime`, which an object validator of the row's fields refuses, and no document ID or generation number leaves the parent (D8, D2, E-9) - validatorReadModelView: `orderSummaryView = v.object({ tenantId: v.string(), key: v.string(), projectionVersion: v.number(), sourceVersions: v.array(streamVersionValidator), ...orderSummaryFields })` where `orderSummaryFields` are the projected fields `spec:application.projection-contract` declares for `tableOrderSummaries` (E-9, D8) diff --git a/example/convex/readModels.ts b/example/convex/readModels.ts index 4777588..39c4270 100644 --- a/example/convex/readModels.ts +++ b/example/convex/readModels.ts @@ -65,7 +65,11 @@ export const listOrderSummaries = query({ ) .paginate( boundedPage( - pageInGeneration(paginationOpts, generation, byStatusFields), + pageInGeneration( + paginationOpts, + [tenantId, generation, status], + byStatusFields, + ), limitReadModelList(orderSummary), ), ); diff --git a/fixture/convex/readModels.ts b/fixture/convex/readModels.ts index 9eb885b..17afdf5 100644 --- a/fixture/convex/readModels.ts +++ b/fixture/convex/readModels.ts @@ -63,7 +63,11 @@ export const listDocumentSummaries = query({ ) .paginate( boundedPage( - pageInGeneration(paginationOpts, generation, byStatusFields), + pageInGeneration( + paginationOpts, + [tenantId, generation, status], + byStatusFields, + ), limitReadModelList(documentSummary), ), ); diff --git a/src/read-model/queries.ts b/src/read-model/queries.ts index 710d61f..73297e6 100644 --- a/src/read-model/queries.ts +++ b/src/read-model/queries.ts @@ -6,6 +6,7 @@ import { type PageLimit, } from "../context/queries.js"; import type { PaginationOptions } from "convex/server"; +import type { Value } from "convex/values"; import type { AnyReadModel } from "./projection.js"; // A read-model row as a query returns it: the document ID, its creation time and the generation stay // in the parent. @@ -22,39 +23,46 @@ export const limitReadModelList = (readModel: AnyReadModel): PageLimit => ({ bytes: limitListBytes, }); // A client's page options with each cursor moved into the generation now active. A cursor of -// paginator is the JSON of a row's index key, whose second element is the generation that wrote the -// row; a key of the index's fields plus two also carries that document's _creationTime and _id, which -// are dropped. A cursor already moved keeps its length, so moving it again replaces the generation alone. +// paginator is the JSON of a row's index key. Its equality prefix must be the query's, apart from the +// generation at the second place, which is replaced; a key of the index's fields plus two also carries +// that document's _creationTime and _id, which are dropped. A cursor already moved keeps its length, so +// moving it again replaces the generation alone. Any other cursor is refused before a read. export function pageInGeneration( opts: PaginationOptions, - generation: number, + prefix: readonly Value[], indexFields: number, ): PaginationOptions { const moved = { ...opts, - cursor: moveCursor(opts.cursor, generation, indexFields), + cursor: moveCursor(opts.cursor, prefix, indexFields), }; if (opts.endCursor !== undefined && opts.endCursor !== null) - moved.endCursor = moveCursor(opts.endCursor, generation, indexFields); + moved.endCursor = moveCursor(opts.endCursor, prefix, indexFields); return moved; } +const generationAt = 1; function moveCursor( cursor: string | null, - generation: number, + prefix: readonly Value[], indexFields: number, ) { - if (cursor === null) return cursor; + // null starts the list; "[]" is paginator's end of the list. + if (cursor === null || cursor === "[]") return cursor; let key: unknown; try { key = JSON.parse(cursor); } catch { - return cursor; + key = undefined; } if ( !Array.isArray(key) || (key.length !== indexFields && key.length !== indexFields + 2) || - typeof key[1] !== "number" + typeof key[generationAt] !== "number" || + prefix.some( + (value, i) => + i !== generationAt && JSON.stringify(key[i]) !== JSON.stringify(value), + ) ) - return cursor; - return JSON.stringify([key[0], generation, ...key.slice(2, indexFields)]); + throw new Error("The cursor is not from this list"); + return JSON.stringify([...prefix, ...key.slice(prefix.length, indexFields)]); } diff --git a/tests/simulator/read-model-page-in-generation.test.ts b/tests/simulator/read-model-page-in-generation.test.ts index e439792..83c2907 100644 --- a/tests/simulator/read-model-page-in-generation.test.ts +++ b/tests/simulator/read-model-page-in-generation.test.ts @@ -1,13 +1,24 @@ -// read-models.sdp.md fnPageInGeneration, indexReadModelList, pagination and the two rules on a -// client's page across a switch; projection-contract.sdp.md tableOrderSummaries. +// read-models.sdp.md fnPageInGeneration, fnListOrSummary, indexReadModelList, pagination and the two +// rules on a client's page across a switch; projection-contract.sdp.md tableOrderSummaries. Every case +// runs through a composition's list entry. import type { PaginationOptions } from "convex/server"; -import { paginator } from "convex-helpers/server/pagination"; import { expect, test } from "vitest"; -import schema from "../../example/convex/schema.js"; -import { pageInGeneration } from "../../src/read-model/index.js"; +import { + api as productionApi, + internal as productionInternal, +} from "../../example/convex/_generated/api.js"; +import { readOrdersPermission } from "../../example/convex/readModels.js"; +import { + api as fixtureApi, + internal as fixtureInternal, +} from "../../fixture/convex/_generated/api.js"; +import { readPermission } from "../../fixture/convex/readModels.js"; import { productionTest } from "./production.js"; +import { rebuildApp } from "./rebuild-support.js"; +const issuer = "https://page.test"; const tenantId = "tenant-a"; +const refused = "The cursor is not from this list"; // b and c share placedAt, so a page edge between them is a tie on the order field. const orders = [ ["a", 1], @@ -16,68 +27,102 @@ const orders = [ ["d", 3], ["e", 4], ] as const; -// tenantId, generation, status, placedAt and key: by_status of orderSummaries. -const byStatusFields = 5; +type Production = ReturnType; type Page = { rows: string[]; isDone: boolean; continueCursor: string }; +const generationRow = (readModel: string, generation: number) => ({ + readModel, + generation, + projectionVersion: 1, + state: generation === 1 ? ("active" as const) : ("verified" as const), + pauseRequired: false, + fence: 1, + startedAt: 0, + startedBy: "operator", + changedAt: 0, + changedBy: "operator", +}); +// Generations 1 and 2 of the order summary hold the same orders in two tenants; a row's total is the +// generation that wrote it, so a page shows which generation it read. Generation 1 is active. async function seeded() { const t = productionTest(); + for (const tenant of [tenantId, "tenant-b"]) + await t.mutation(productionInternal.grants.grant, { + tenantId: tenant, + principalKind: "human", + principalId: `${issuer}|reader`, + permission: readOrdersPermission, + grantedBy: "operator", + }); await t.run(async (ctx) => { - for (const generation of [1, 2]) - for (const [key, placedAt] of orders) - await ctx.db.insert("orderSummaries", { - tenantId, - generation, - key, - projectionVersion: 1, - sourceVersions: [], - orderId: key, - status: "placed", - lineCount: 1, - total: 1, - placedAt, - }); + for (const generation of [1, 2]) { + await ctx.db.insert( + "generations", + generationRow("orderSummary", generation), + ); + for (const tenant of [tenantId, "tenant-b"]) + for (const [key, placedAt] of orders) + await ctx.db.insert("orderSummaries", { + tenantId: tenant, + generation, + key, + projectionVersion: 1, + sourceVersions: [], + orderId: key, + status: "placed", + lineCount: 1, + total: generation, + placedAt, + }); + } }); return t; } -// One page of the list as listOrderSummaries reads it, in the generation then active. -function page( - t: Awaited>, - generation: number, +// The switch as the registry records it: the named generation active, the other retired. +async function activate(t: Production, generation: number) { + await t.run(async (ctx) => { + for (const row of await ctx.db.query("generations").collect()) + await ctx.db.patch(row._id, { + state: row.generation === generation ? "active" : "retired", + }); + }); +} +async function page( + t: Production, opts: PaginationOptions, + args: { tenantId?: string; status?: "placed" | "cancelled" } = {}, ): Promise { - return t.run(async (ctx) => { - const result = await paginator(ctx.db, schema) - .query("orderSummaries") - .withIndex("by_status", (q) => - q - .eq("tenantId", tenantId) - .eq("generation", generation) - .eq("status", "placed"), - ) - .paginate(pageInGeneration(opts, generation, byStatusFields)); - return { - rows: result.page.map((row) => `${row.key}@${row.generation}`), - isDone: result.isDone, - continueCursor: result.continueCursor, - }; - }); + const result = await t + .withIdentity({ issuer, subject: "reader" }) + .query(productionApi.readModels.listOrderSummaries, { + tenantId, + status: "placed", + ...args, + paginationOpts: opts, + }); + return { + rows: result.page.map((row) => `${row.key}@${row.total}`), + isDone: result.isDone, + continueCursor: result.continueCursor, + }; } test("convex-test: a later page's cursor continues from the same position in the generation switched to", async () => { const t = await seeded(); - const first = await page(t, 1, { cursor: null, numItems: 1 }); + const first = await page(t, { cursor: null, numItems: 1 }); expect(first.rows).toEqual(["a@1"]); + await activate(t, 2); expect( - (await page(t, 2, { cursor: first.continueCursor, numItems: 2 })).rows, + (await page(t, { cursor: first.continueCursor, numItems: 2 })).rows, ).toEqual(["b@2", "c@2"]); }); test("convex-test: a pinned page shows the switched-to generation's rows up to its end cursor", async () => { const t = await seeded(); - const first = await page(t, 1, { cursor: null, numItems: 2 }); + const first = await page(t, { cursor: null, numItems: 2 }); expect(first.rows).toEqual(["a@1", "b@1"]); - const pinned = await page(t, 2, { + await activate(t, 2); + const pinned = await page(t, { cursor: null, endCursor: first.continueCursor, numItems: 2, @@ -86,53 +131,118 @@ test("convex-test: a pinned page shows the switched-to generation's rows up to i expect(pinned.isDone).toBe(false); }); -test("convex-test: a pinned page's continueCursor, already moved, pages on across a switch", async () => { +test("convex-test: a pinned page's continueCursor, already moved, pages on across a switch back", async () => { const t = await seeded(); - const first = await page(t, 1, { cursor: null, numItems: 2 }); - const pinned = await page(t, 2, { + const first = await page(t, { cursor: null, numItems: 2 }); + await activate(t, 2); + const pinned = await page(t, { cursor: null, endCursor: first.continueCursor, numItems: 2, }); expect(pinned.rows).toEqual(["a@2", "b@2"]); + await activate(t, 1); expect( - (await page(t, 1, { cursor: pinned.continueCursor, numItems: 2 })).rows, + (await page(t, { cursor: pinned.continueCursor, numItems: 2 })).rows, ).toEqual(["c@1", "d@1"]); }); test("convex-test: after a switch back a cursor of generation 2 continues from the same position in generation 1", async () => { const t = await seeded(); - const first = await page(t, 2, { cursor: null, numItems: 2 }); + await activate(t, 2); + const first = await page(t, { cursor: null, numItems: 2 }); expect(first.rows).toEqual(["a@2", "b@2"]); + await activate(t, 1); expect( - (await page(t, 1, { cursor: first.continueCursor, numItems: 2 })).rows, + (await page(t, { cursor: first.continueCursor, numItems: 2 })).rows, ).toEqual(["c@1", "d@1"]); }); test("convex-test: a page edge on an order-field tie neither repeats nor skips a row across a switch", async () => { const t = await seeded(); - const first = await page(t, 1, { cursor: null, numItems: 2 }); + const first = await page(t, { cursor: null, numItems: 2 }); expect(first.rows).toEqual(["a@1", "b@1"]); - const next = await page(t, 2, { - cursor: first.continueCursor, - numItems: 2, - }); - expect(next.rows).toEqual(["c@2", "d@2"]); + await activate(t, 2); + expect( + (await page(t, { cursor: first.continueCursor, numItems: 2 })).rows, + ).toEqual(["c@2", "d@2"]); }); -test("convex-test: pageInGeneration returns a cursor that is not an index key as it is", () => { - for (const cursor of [null, "[]", "not json", '{"a":1}', '["a","b","c","d"]']) - expect( - pageInGeneration({ cursor, numItems: 1 }, 2, byStatusFields), - ).toStrictEqual({ - cursor, - numItems: 1, +test("convex-test: a cursor of another tenant, another status, the wrong length or a non-numeric generation is refused", async () => { + const t = await seeded(); + const other = await page( + t, + { cursor: null, numItems: 2 }, + { tenantId: "tenant-b" }, + ); + const key = JSON.parse(other.continueCursor) as unknown[]; + const cursors = [ + other.continueCursor, + JSON.stringify([tenantId, 1, "cancelled", ...key.slice(3)]), + JSON.stringify([tenantId, 1, "placed", ...key.slice(3, 6)]), + JSON.stringify([tenantId, "1", "placed", ...key.slice(3)]), + "not json", + ]; + for (const cursor of cursors) { + await expect(page(t, { cursor, numItems: 2 })).rejects.toThrow(refused); + await expect( + page(t, { cursor: null, endCursor: cursor, numItems: 2 }), + ).rejects.toThrow(refused); + } +}); + +test("convex-test: the fixture's document list moves a cursor across a switch and refuses another tenant's", async () => { + const t = rebuildApp(); + for (const tenant of [tenantId, "tenant-b"]) + await t.mutation(fixtureInternal.grants.grant, { + tenantId: tenant, + principalKind: "human", + principalId: `${issuer}|reader`, + permission: readPermission, + grantedBy: "operator", }); + await t.run(async (ctx) => { + for (const generation of [1, 2]) { + await ctx.db.insert( + "generations", + generationRow("documentSummary", generation), + ); + for (const tenant of [tenantId, "tenant-b"]) + for (const [key] of orders) + await ctx.db.insert("documentSummaries", { + tenantId: tenant, + generation, + key, + projectionVersion: 1, + sourceVersions: [], + documentId: key, + status: "draft", + title: `${key}@${generation}`, + }); + } + }); + const list = async (tenant: string, opts: PaginationOptions) => { + const result = await t + .withIdentity({ issuer, subject: "reader" }) + .query(fixtureApi.readModels.listDocumentSummaries, { + tenantId: tenant, + status: "draft", + paginationOpts: opts, + }); + return { ...result, rows: result.page.map(({ title }) => title) }; + }; + const first = await list(tenantId, { cursor: null, numItems: 2 }); + expect(first.rows).toEqual(["a@1", "b@1"]); + await t.run(async (ctx) => { + for (const row of await ctx.db.query("generations").collect()) + await ctx.db.patch(row._id, { + state: row.generation === 2 ? "active" : "retired", + }); + }); expect( - pageInGeneration( - { cursor: null, endCursor: null, numItems: 1 }, - 2, - byStatusFields, - ), - ).toStrictEqual({ cursor: null, endCursor: null, numItems: 1 }); + (await list(tenantId, { cursor: first.continueCursor, numItems: 2 })).rows, + ).toEqual(["c@2", "d@2"]); + await expect( + list("tenant-b", { cursor: first.continueCursor, numItems: 2 }), + ).rejects.toThrow(refused); }); From 8ba134239376a20385785174e9bfdf92a28dd1dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 05:48:23 +0200 Subject: [PATCH 104/155] design: 3b the seam, the pinned page's caps and the first rebuild's order The seam shows no row twice and skips none when both projections give a subject the same order field and key; a pinned page's caps are paginator's, the byte cap admitting its last row whole; the first-rebuild example orders its steps as the test runs them. Co-Authored-By: Claude Opus 5.5 --- ...ration-registry.first-rebuild-installs-read-model.sdp.md | 2 +- design/specs/application/read-models.sdp.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md b/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md index 01045b3..83ac714 100644 --- a/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md +++ b/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md @@ -27,6 +27,6 @@ And a second start while generation 1 is being built is {second: "refused"} - Runs in the native tier on the fixture composition; every test owns its disposable backend. - The test creates three documents through `CreateDocument`, which declares no read model, then runs `internal.rebuild.startGeneration` for `documentSummary` with admin access and a stated operator, and asserts one row in `generations`, generation 1, `building`, with `documentSummary`'s latest projection version and the operator as `startedBy`. -- The test closes the tenant's gate before the first start, so the first batch parks under it, runs `startGeneration` again before the first generation is switched and asserts that its error names the read model, generation 1 and its state, and that the table still holds one row, then reopens the gate and runs `resumeGeneration`. +- In this order the test closes the tenant's gate, runs the first start, reads the generation row `building`, runs `startGeneration` again and asserts that its error names the read model, generation 1 and its state and that the table still holds one row, waits until the first batch has parked under the gate, reopens the gate and runs `resumeGeneration`. - The test waits until the row is `verified`, asserts that `listDocumentSummaries` still fails for want of an active generation, runs `switchGeneration`, and asserts that the row is `active`, that no row is `retired`, and that `listDocumentSummaries` returns the three documents under their status. - The test runs the same sequence on a backend with one tenant and no subject and asserts that the progress row's `batchesDone` is 2, one backfill batch and one verify batch, before `verified`. diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index 63b559c..20b3dbc 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -46,7 +46,7 @@ This Spec carries the read-need table as rules, the properties of a projection, ### Open questions -- [non-blocking] Probe 5 showed that the built-in `.paginate` treats `maximumRowsRead` as advice once a page has an end cursor and that a cursor it issued is refused by any other query, so a subscription over the active generation threw `InvalidCursor` at a switch; a parent list therefore pages with `paginator` of `convex-helpers`, as a context list does, and a pinned page is bounded by the row cap and the byte cap on every run and answers `SplitRequired` beyond them (Probe 5, F13, D8, D9) +- [non-blocking] Probe 5 showed that the built-in `.paginate` treats `maximumRowsRead` as advice once a page has an end cursor and that a cursor it issued is refused by any other query, so a subscription over the active generation threw `InvalidCursor` at a switch; a parent list therefore pages with `paginator` of `convex-helpers`, as a context list does, and a pinned page is bounded on every run by the row cap and by the byte cap as `spec:context.queries` states it, the helper stopping at the row that reaches it and admitting that row whole, and answers `SplitRequired` beyond them (Probe 5, F13, D8, D9) - [non-blocking] Extension E-24, as it reaches read models: a list over a parent read-model table calls `paginator` of `convex-helpers` with the options of `boundedPage`, a row cap of twice the item cap, with each cursor of the client's pair moved into the active generation by `pageInGeneration`, and returns through `paginationResultValidator` with its rows mapped by `readModelView`, and a client pages every list by an explicit cursor pair with no hook; these are options taken here and the owner has not ruled (E-24, S4, Probe 5, D8) - [non-blocking] Extension E-6, as it reaches queries: a parent query establishes and authorizes its actor through `authorizeQuery`, and its refusal is the command's rejection shape, `RejectionData` with the query's name in the field `commandType` and the codes `unauthenticated` or `forbidden`; the field keeps its name for a query, which the owner has not ruled on, and nothing else of the shape is new (E-6, D7, D11, Law 5) - [non-blocking] Probe 3 pending: the cost of one component call in latency and function-call quota decides whether one entity's detail stays a component query or hot reads move, which is OQ1 (Probe 3, F4, OQ1) @@ -75,7 +75,7 @@ This Spec carries the read-need table as rules, the properties of a projection, - rule: A parent query establishes and authorizes its actor through `authorizeQuery` before its first read, then reads read-model rows by index or calls context queries with the tenant as argument; a refusal is a rejection and nothing is disclosed (D11, F11, Law 5, D7) - rule: A subscription synchronizes state; it is not durable event delivery, and no consumer treats a query result as proof of having seen every event (F7, D2) - rule: A client's page of a read-model list names its position by the rows' order field and `key` within the tenant and never by the generation, so across a switch and a switch back the first page and every later page show the active generation's rows from the same position, a pinned page the rows up to its end cursor, and no client restarts its paging (D9, D8, E-24) -- rule: A page is read in one transaction, so its rows are one generation's; a cursor used after a switch continues from the same position in the generation then active, so at the seam no row is shown twice or skipped and a row the new generation holds differently appears as that generation holds it (D9, D8, F7) +- rule: A page is read in one transaction, so its rows are one generation's; a cursor used after a switch continues from the same position in the generation then active, so at the seam no row is shown twice or skipped when the two generations' projections give each subject the same order field and key, and a row the new generation holds differently appears as that generation holds it; a projection version that changes either is its author's choice, and a client may then see a row on both sides of the seam or on neither (D9, D8, F7) - rule: [extension] A parent query makes at most 8 component reads, and a command writes at most 4 read-model rows; a read that needs more is a read model or is outside the baseline (E-45, D8) - flow: A client subscribes to a parent query with its tenant scope and the key of the entity, list or view it needs (D8, Law 11) - flow: The query authenticates and authorizes in the parent and returns nothing before authorization passes (Law 5, D11) @@ -105,7 +105,7 @@ Nothing here schedules, and nothing here reads `ctx.auth` inside a component. Th - indexReadModelByKey: `.index("by_key", ["tenantId", "generation", "key"])` on every read-model table (E-9, Law 11) - indexReadModelByKeyUse: point read of one row by the parent query and by `applyProjection` before an update; the generation comes from the registry's active row, which is one per read model and the same for every tenant, so a query narrows by its own `tenantId` and never by a tenant's generation (D8, D9, E-8, Law 11) - pagination: every list returns through `paginationResultValidator(item)` of `convex/server` and passes its one `paginate` call the options of `boundedPage`, as `spec:context.queries` pins; a list over a parent read-model table calls `paginator` of `convex-helpers` with the composition's schema, as a context list does, over the index of `indexReadModelList`, with the client's cursors moved into the active generation by `pageInGeneration`, relays `pageStatus` and `splitCursor` as the helper returns them, and maps the page's rows through `readModelView`; a list served by a context relays the context's page; a client pages either by an explicit cursor pair; a function runs at most one built-in `.paginate`, because Convex refuses a second with "Convex only supports a single paginated query in each function" (D8, S4, Probe 5, F15, E-24) -- limitReadModelList: `limitReadModelList(readModel) = { items: limitListPage(readModel.rowBudgetBytes), bytes: limitListBytes }`, a function of the read model, so 100 items at the default 16 KiB and 64 at the 64 KiB cap; the built-in honours the item cap on a first run and answers `SplitRecommended` for a pinned page that has outgrown the row cap (F13, E-9, Probe 5) +- limitReadModelList: `limitReadModelList(readModel) = { items: limitListPage(readModel.rowBudgetBytes), bytes: limitListBytes }`, a function of the read model, so 100 items at the default 16 KiB and 64 at the 64 KiB cap; `paginator` honours the item cap on a first run and the row cap and the byte cap on every run, and answers `SplitRequired` with a `splitCursor` for a page that reaches either cap (F13, E-9, Probe 5) - limitComponentReadsPerQuery: `[extension]` 8 component reads per parent query (E-45, D8) - limitReadModelWritesPerCommand: `[extension]` 4 read-model rows per command, counted as the rows step 9 inserts, replaces or deletes, a row in each generation once, and checked after each `streams` entry is applied; above it step 9 throws a plain error and nothing of the command commits (E-45, D8, Sc L2-3) - readModelVisibility: a subscription re-runs when a row it read changes, including on generation switch, because the query reads the registry row too; after the switch and after a switch back each page shows the rows of the generation then active from the position its cursors name, so a client sees the same orders and never `InvalidCursor` (D8, F7, D9) From 4ab60bd9ca623d72a360485d64441be8c593606b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 05:48:40 +0200 Subject: [PATCH 105/155] build: 3b fold-in getGenerations lists only a declared read model Co-Authored-By: Claude Opus 5.5 --- src/read-model/rebuild.ts | 7 ++++++- tests/simulator/rebuild.test.ts | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/read-model/rebuild.ts b/src/read-model/rebuild.ts index 8f6f92c..60d64f6 100644 --- a/src/read-model/rebuild.ts +++ b/src/read-model/rebuild.ts @@ -631,9 +631,14 @@ export async function purgeBatch( } export async function getGenerations( parentCtx: GenericQueryCtx, - _config: RebuildConfig, + config: RebuildConfig, args: { readModel: string }, ) { + // rebuild.sdp.md:137: a read model the composition does not declare has no generations to list. + if ( + !config.targets.some(({ readModel }) => readModel.name === args.readModel) + ) + return []; const ctx = parentCtx as unknown as GenericQueryCtx; const generations = await ctx.db .query("generations") diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts index 353c5d7..ce50d47 100644 --- a/tests/simulator/rebuild.test.ts +++ b/tests/simulator/rebuild.test.ts @@ -2347,3 +2347,17 @@ test("convex-test: startGeneration schedules exactly one batch and an install le await t.run((ctx) => ctx.db.query("projectionMarkers").collect()), ).toStrictEqual([]); }); + +// rebuild.sdp.md fnGetGenerations:137: only a declared read model's rows are listed. +test("convex-test: getGenerations answers [] for an undeclared read model even when rows name it", async () => { + const t = rebuildApp(); + const id = await start(t); + await t.run(async (ctx) => { + const { _id, _creationTime, ...row } = (await ctx.db.get(id))!; + void [_id, _creationTime]; + await ctx.db.insert("generations", { ...row, readModel: "undeclared" }); + }); + expect( + await t.query(internal.rebuild.getGenerations, { readModel: "undeclared" }), + ).toStrictEqual([]); +}); From 2283083146c9af7e9257b6f61ed115bec76471e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 05:49:04 +0200 Subject: [PATCH 106/155] build: 3b fold-in rollbackGeneration's refusal order pinned with a pause and a fill together Co-Authored-By: Claude Opus 5.5 --- tests/simulator/rebuild.test.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts index ce50d47..6bc1db7 100644 --- a/tests/simulator/rebuild.test.ts +++ b/tests/simulator/rebuild.test.ts @@ -1263,19 +1263,9 @@ test("convex-test: rollback checks period, purge, pause, in-flight generation an "Generation 1 of documentTitle is already being purged", ); await patchGeneration(t, old, {}, { pass: "idle" }); - await refused( - t, - () => - t.mutation(internal.rebuild.rollbackGeneration, { - generationId: old, - operator, - }), - "Generation 1 of documentTitle needs a write pause and cannot be rolled back", - ); - await patchGeneration(t, old, { pauseRequired: false }); const other = await start(t); - // rebuild.sdp.md:84: a fill and a reopened generation exclude each other; the fill refusal - // precedes the in-flight refusal. + // rebuild.sdp.md:84, :134: a fill and a reopened generation exclude each other; the pause refusal + // precedes the fill refusal, which precedes the in-flight refusal. const fillRow = { fence: 1, cursor: null, @@ -1293,6 +1283,16 @@ test("convex-test: rollback checks period, purge, pause, in-flight generation an if (prior !== null) await ctx.db.delete(prior._id); return ctx.db.insert("tenantFill", { ...fillRow, pass: "fill" }); }); + await refused( + t, + () => + t.mutation(internal.rebuild.rollbackGeneration, { + generationId: old, + operator, + }), + "Generation 1 of documentTitle needs a write pause and cannot be rolled back", + ); + await patchGeneration(t, old, { pauseRequired: false }); await refused( t, () => From d149beb376b643abb72ee03b6ba15c405164f749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 05:59:31 +0200 Subject: [PATCH 107/155] build: 3b fold-in the cursor read and written as paginator encodes it pageInGeneration decodes a cursor's key with paginator's own codec, undefined as "undefined" and a string ending in undefined escaped by one underscore, compares and replaces in decoded values and encodes back, so a tenant t-undefined pages on from its own cursor and _t-undefined is refused it. The codec is held to a cursor paginator issues; the last pinned page's end cursor "[]" passes and the refusal is a plain Error. The Spec's fnPageInGeneration says so, with what passes as a start or an end cursor and where the refusal falls. Co-Authored-By: Claude Opus 5.5 --- design/specs/application/read-models.sdp.md | 2 +- src/read-model/queries.ts | 51 ++++++++-- .../read-model-page-in-generation.test.ts | 93 ++++++++++++++++++- 3 files changed, 132 insertions(+), 14 deletions(-) diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index 20b3dbc..b6d05d9 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -94,7 +94,7 @@ Nothing here schedules, and nothing here reads `ctx.auth` inside a component. Th - convexSurface: one static `query` export per read need in the parent; component queries are reached through `ctx.runQuery(components.orders.queries.order.get, args)`, its siblings `list`, `history` and `rebuild`, and `components.orders.queries.operations.byOperation`, as the queries contract names them; no scheduled function, no cron, no action (D8, F11) - fnEntityDetail: `getOrder` as `spec:context.queries` pins it under `parentGet`: it calls `authorizeQuery` and then `ctx.runQuery(components.orders.queries.order.get, { tenantId, streamId: orderId })` (D8, D11, F15) - fnListOrSummary: `export const listOrderSummaries = query({ args: { tenantId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(orderSummaryView), handler: async (ctx, { tenantId, status, paginationOpts }) => { await authorizeQuery(ctx, { name: "listOrderSummaries", tenantId, permission: "orders.read" }); const generation = await activeGeneration(ctx, "orderSummary"); if (generation === undefined) throw new Error("The read model orderSummary has no active generation"); const result = await paginator(ctx.db, schema).query("orderSummaries").withIndex("by_status", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("status", status)).paginate(boundedPage(pageInGeneration(paginationOpts, [tenantId, generation, status], 5), limitReadModelList(orderSummary))); return { ...result, page: result.page.map(readModelView) }; } })` one `paginate` of `paginator` from `convex-helpers/server/pagination` with the composition's `schema`, over the active generation's rows in `placedAt` then `key` order within a status, where `orderSummary` is the read model, whose name is `"orderSummary"` and whose table is `orderSummaries` (D8, D9, Law 5, Law 11, E-24) -- fnPageInGeneration: `pageInGeneration(opts: PaginationOptions, prefix: Value[], indexFields: number): PaginationOptions` returns `opts` with its `cursor` and its `endCursor`, when set, each moved into the generation of `prefix`, the query's equality values in index order, `tenantId`, `generation` and the list's equality fields, the generation second: a cursor of `paginator` is the JSON of a row's index key, so the function replaces the key's second element, the generation that wrote the row, with the prefix's and drops the last two, `_creationTime` and `_id`, which belong to that generation's document, when the key carries them, which it tells by a length of `indexFields`, the index's own field count, plus two, so that a cursor already moved, which `paginator` returns as a pinned page's `continueCursor`, keeps its length and has only its generation replaced; a cursor whose key's length is neither `indexFields` nor `indexFields` plus two, whose generation is not a number, whose other equality elements differ from the prefix's, or that is not a key, throws a plain `Error` `"The cursor is not from this list"` before the list reads a row, so a saved cursor never overrides the query's tenant or status; `null`, `undefined` and `"[]"`, the helper's end of the list, are returned as they are, and the read-model library exports the function beside `readModelView` (D9, E-24, Probe 5, Law 11) +- fnPageInGeneration: `pageInGeneration(opts: PaginationOptions, prefix: Value[], indexFields: number): PaginationOptions` returns `opts` with its `cursor` and its `endCursor`, when set, each moved into the generation of `prefix`, the query's equality values in index order, `tenantId`, `generation` and the list's equality fields, the generation second: a cursor of `paginator` is the JSON of a row's index key, which the function reads and writes as `paginator` encodes it, `undefined` as the string `"undefined"` and a string that ends in `undefined` with one leading underscore added, so it compares the equality elements and replaces the generation in the decoded values; it replaces the key's second element, the generation that wrote the row, with the prefix's and drops the last two, `_creationTime` and `_id`, which belong to that generation's document, when the key carries them, which it tells by a length of `indexFields`, the index's own field count, plus two, so that a cursor already moved, which `paginator` returns as a pinned page's `continueCursor`, keeps its length and has only its generation replaced; a cursor whose key's length is neither `indexFields` nor `indexFields` plus two, whose generation is not a number, whose other equality elements differ from the prefix's, or that is not a key, throws a plain `Error` `"The cursor is not from this list"`, after `authorizeQuery` and the read of the active generation and before the list reads a row, so a saved cursor never overrides the query's tenant or status; `null` passes as a start or an end cursor and `undefined` as an end cursor, the validator requiring a start cursor, and `"[]"`, the helper's end-of-list mark, passes as the lawful `endCursor` of the last pinned page and, as a start cursor, reads the list from its first row, which is the helper's behavior and discloses nothing; the read-model library exports the function beside `readModelView` (D9, E-24, Probe 5, Law 11) - indexReadModelList: the index a read-model list reads has `tenantId`, `generation` and the list's equality fields first, then its order field, then `key`, as `by_status` of `tableOrderSummaries` in `spec:application.projection-contract`, so a position within a tenant names at most one row of each generation and a cursor moved into another generation lands between the same rows there (D9, E-9, Law 11) - fnReadModelView: `readModelView(row: Row): Omit` what a read-model query returns for a row: the tenant, the key, the projection version, the source versions and the projected fields; the built-in page's rows carry `_id` and `_creationTime`, which an object validator of the row's fields refuses, and no document ID or generation number leaves the parent (D8, D2, E-9) - validatorReadModelView: `orderSummaryView = v.object({ tenantId: v.string(), key: v.string(), projectionVersion: v.number(), sourceVersions: v.array(streamVersionValidator), ...orderSummaryFields })` where `orderSummaryFields` are the projected fields `spec:application.projection-contract` declares for `tableOrderSummaries` (E-9, D8) diff --git a/src/read-model/queries.ts b/src/read-model/queries.ts index 73297e6..d2db04f 100644 --- a/src/read-model/queries.ts +++ b/src/read-model/queries.ts @@ -6,7 +6,7 @@ import { type PageLimit, } from "../context/queries.js"; import type { PaginationOptions } from "convex/server"; -import type { Value } from "convex/values"; +import { convexToJson, jsonToConvex, type Value } from "convex/values"; import type { AnyReadModel } from "./projection.js"; // A read-model row as a query returns it: the document ID, its creation time and the generation stay // in the parent. @@ -41,28 +41,59 @@ export function pageInGeneration( return moved; } const generationAt = 1; +// paginator's own cursor codec, which it does not export (convex-helpers 0.1.124, +// server/stream.js:1419 to :1441): undefined is written as the string "undefined", a string that ends +// in "undefined" gains one leading underscore, and the key goes through convexToJson. The two below +// read and write a key exactly so, and the comparison and the generation's replacement happen on the +// decoded values. +export function decodeCursorKey(cursor: string): Value[] { + const json: unknown = JSON.parse(cursor); + if (!Array.isArray(json)) throw new Error("not a key"); + return (jsonToConvex(json) as Value[]).map((value) => + typeof value === "string" && value.endsWith("undefined") + ? value === "undefined" + ? (undefined as unknown as Value) + : value.slice(1) + : value, + ); +} +export function encodeCursorKey(key: readonly Value[]): string { + return JSON.stringify( + convexToJson( + key.map((value) => + value === undefined + ? "undefined" + : typeof value === "string" && value.endsWith("undefined") + ? "_" + value + : value, + ), + ), + ); +} +const sameValue = (a: Value | undefined, b: Value | undefined) => + a === b || + (a !== undefined && + b !== undefined && + JSON.stringify(convexToJson(a)) === JSON.stringify(convexToJson(b))); function moveCursor( cursor: string | null, prefix: readonly Value[], indexFields: number, ) { - // null starts the list; "[]" is paginator's end of the list. + // null starts the list; "[]" is paginator's end of the list, and as a start it reads from the first row. if (cursor === null || cursor === "[]") return cursor; - let key: unknown; + let key: Value[] | undefined; try { - key = JSON.parse(cursor); + key = decodeCursorKey(cursor); } catch { key = undefined; } if ( - !Array.isArray(key) || + key === undefined || (key.length !== indexFields && key.length !== indexFields + 2) || typeof key[generationAt] !== "number" || - prefix.some( - (value, i) => - i !== generationAt && JSON.stringify(key[i]) !== JSON.stringify(value), - ) + prefix.some((value, i) => i !== generationAt && !sameValue(key[i], value)) ) throw new Error("The cursor is not from this list"); - return JSON.stringify([...prefix, ...key.slice(prefix.length, indexFields)]); + return encodeCursorKey([...prefix, ...key.slice(prefix.length, indexFields)]); } diff --git a/tests/simulator/read-model-page-in-generation.test.ts b/tests/simulator/read-model-page-in-generation.test.ts index 83c2907..f241065 100644 --- a/tests/simulator/read-model-page-in-generation.test.ts +++ b/tests/simulator/read-model-page-in-generation.test.ts @@ -2,7 +2,14 @@ // rules on a client's page across a switch; projection-contract.sdp.md tableOrderSummaries. Every case // runs through a composition's list entry. import type { PaginationOptions } from "convex/server"; +import { ConvexError } from "convex/values"; +import { paginator } from "convex-helpers/server/pagination"; import { expect, test } from "vitest"; +import schema from "../../example/convex/schema.js"; +import { + decodeCursorKey, + encodeCursorKey, +} from "../../src/read-model/queries.js"; import { api as productionApi, internal as productionInternal, @@ -44,9 +51,9 @@ const generationRow = (readModel: string, generation: number) => ({ // Generations 1 and 2 of the order summary hold the same orders in two tenants; a row's total is the // generation that wrote it, so a page shows which generation it read. Generation 1 is active. -async function seeded() { +async function seeded(tenants: readonly string[] = [tenantId, "tenant-b"]) { const t = productionTest(); - for (const tenant of [tenantId, "tenant-b"]) + for (const tenant of tenants) await t.mutation(productionInternal.grants.grant, { tenantId: tenant, principalKind: "human", @@ -60,7 +67,7 @@ async function seeded() { "generations", generationRow("orderSummary", generation), ); - for (const tenant of [tenantId, "tenant-b"]) + for (const tenant of tenants) for (const [key, placedAt] of orders) await ctx.db.insert("orderSummaries", { tenantId: tenant, @@ -246,3 +253,83 @@ test("convex-test: the fixture's document list moves a cursor across a switch an list("tenant-b", { cursor: first.continueCursor, numItems: 2 }), ).rejects.toThrow(refused); }); + +test("convex-test: the cursor codec reads and writes a key as paginator does", async () => { + for (const value of ["undefined", "_undefined", "t-undefined", "plain", 3]) + expect(decodeCursorKey(encodeCursorKey([value, 1]))).toStrictEqual([ + value, + 1, + ]); + expect(encodeCursorKey(["t-undefined", undefined as never])).toBe( + '["_t-undefined","undefined"]', + ); + // A cursor paginator itself issues for a row whose tenant ends in "undefined". + const t = await seeded(["t-undefined"]); + const issued = await t.run(async (ctx) => { + const result = await paginator(ctx.db, schema) + .query("orderSummaries") + .withIndex("by_status", (q) => + q + .eq("tenantId", "t-undefined") + .eq("generation", 1) + .eq("status", "placed"), + ) + .paginate({ cursor: null, numItems: 1 }); + return result.continueCursor; + }); + const key = decodeCursorKey(issued); + expect(key.slice(0, 5)).toStrictEqual(["t-undefined", 1, "placed", 1, "a"]); + expect(encodeCursorKey(key)).toBe(issued); +}); + +test("convex-test: a tenant whose ID ends in undefined pages on from its own cursor and no other tenant reads it", async () => { + const t = await seeded(["t-undefined", "_t-undefined"]); + const first = await page( + t, + { cursor: null, numItems: 2 }, + { tenantId: "t-undefined" }, + ); + expect(first.rows).toEqual(["a@1", "b@1"]); + await activate(t, 2); + expect( + ( + await page( + t, + { cursor: first.continueCursor, numItems: 2 }, + { tenantId: "t-undefined" }, + ) + ).rows, + ).toEqual(["c@2", "d@2"]); + await expect( + page( + t, + { cursor: first.continueCursor, numItems: 2 }, + { tenantId: "_t-undefined" }, + ), + ).rejects.toThrow(refused); +}); + +test('convex-test: the last pinned page\'s end cursor "[]" is accepted and a refusal is a plain Error', async () => { + const t = await seeded(); + const all = await page(t, { cursor: null, numItems: 10 }); + expect(all.isDone).toBe(true); + expect(all.continueCursor).toBe("[]"); + await activate(t, 2); + const pinned = await page(t, { + cursor: null, + endCursor: all.continueCursor, + numItems: 10, + }); + expect(pinned.rows).toEqual(["a@2", "b@2", "c@2", "d@2", "e@2"]); + expect(pinned.isDone).toBe(true); + const error: unknown = await page(t, { + cursor: "not json", + numItems: 2, + }).then( + () => undefined, + (e: unknown) => e, + ); + expect(error).toBeInstanceOf(Error); + expect(error).not.toBeInstanceOf(ConvexError); + expect((error as { data?: unknown }).data).toBeUndefined(); +}); From de285bba1e89bf79841ed56aaad833257702e34f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 05:59:31 +0200 Subject: [PATCH 108/155] design: 3b the seam's qualification names the status Co-Authored-By: Claude Opus 5.5 --- design/specs/application/read-models.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index b6d05d9..9991515 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -75,7 +75,7 @@ This Spec carries the read-need table as rules, the properties of a projection, - rule: A parent query establishes and authorizes its actor through `authorizeQuery` before its first read, then reads read-model rows by index or calls context queries with the tenant as argument; a refusal is a rejection and nothing is disclosed (D11, F11, Law 5, D7) - rule: A subscription synchronizes state; it is not durable event delivery, and no consumer treats a query result as proof of having seen every event (F7, D2) - rule: A client's page of a read-model list names its position by the rows' order field and `key` within the tenant and never by the generation, so across a switch and a switch back the first page and every later page show the active generation's rows from the same position, a pinned page the rows up to its end cursor, and no client restarts its paging (D9, D8, E-24) -- rule: A page is read in one transaction, so its rows are one generation's; a cursor used after a switch continues from the same position in the generation then active, so at the seam no row is shown twice or skipped when the two generations' projections give each subject the same order field and key, and a row the new generation holds differently appears as that generation holds it; a projection version that changes either is its author's choice, and a client may then see a row on both sides of the seam or on neither (D9, D8, F7) +- rule: A page is read in one transaction, so its rows are one generation's; a cursor used after a switch continues from the same position in the generation then active, so at the seam no row is shown twice or skipped when the two generations' projections give each subject the same order field, key and status, and a row the new generation holds differently appears as that generation holds it; a projection version that changes either is its author's choice, and a client may then see a row on both sides of the seam or on neither (D9, D8, F7) - rule: [extension] A parent query makes at most 8 component reads, and a command writes at most 4 read-model rows; a read that needs more is a read model or is outside the baseline (E-45, D8) - flow: A client subscribes to a parent query with its tenant scope and the key of the entity, list or view it needs (D8, Law 11) - flow: The query authenticates and authorizes in the parent and returns nothing before authorization passes (Law 5, D11) From 399b41098fb238b4bf185bec92e615aad4ddf0c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 06:22:39 +0200 Subject: [PATCH 109/155] design: 3b the disclosure sentence names the cursor The review's last text gap: a page's cursor carries the generation and, as paginator issues it, the row's _creationTime and _id, which the sentence on what leaves the parent now says; a client passes a cursor back unread and the list refuses one that is not its own. Co-Authored-By: Claude Fable 5.1 --- design/specs/application/read-models.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index 9991515..60eddb5 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -96,7 +96,7 @@ Nothing here schedules, and nothing here reads `ctx.auth` inside a component. Th - fnListOrSummary: `export const listOrderSummaries = query({ args: { tenantId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(orderSummaryView), handler: async (ctx, { tenantId, status, paginationOpts }) => { await authorizeQuery(ctx, { name: "listOrderSummaries", tenantId, permission: "orders.read" }); const generation = await activeGeneration(ctx, "orderSummary"); if (generation === undefined) throw new Error("The read model orderSummary has no active generation"); const result = await paginator(ctx.db, schema).query("orderSummaries").withIndex("by_status", (q) => q.eq("tenantId", tenantId).eq("generation", generation).eq("status", status)).paginate(boundedPage(pageInGeneration(paginationOpts, [tenantId, generation, status], 5), limitReadModelList(orderSummary))); return { ...result, page: result.page.map(readModelView) }; } })` one `paginate` of `paginator` from `convex-helpers/server/pagination` with the composition's `schema`, over the active generation's rows in `placedAt` then `key` order within a status, where `orderSummary` is the read model, whose name is `"orderSummary"` and whose table is `orderSummaries` (D8, D9, Law 5, Law 11, E-24) - fnPageInGeneration: `pageInGeneration(opts: PaginationOptions, prefix: Value[], indexFields: number): PaginationOptions` returns `opts` with its `cursor` and its `endCursor`, when set, each moved into the generation of `prefix`, the query's equality values in index order, `tenantId`, `generation` and the list's equality fields, the generation second: a cursor of `paginator` is the JSON of a row's index key, which the function reads and writes as `paginator` encodes it, `undefined` as the string `"undefined"` and a string that ends in `undefined` with one leading underscore added, so it compares the equality elements and replaces the generation in the decoded values; it replaces the key's second element, the generation that wrote the row, with the prefix's and drops the last two, `_creationTime` and `_id`, which belong to that generation's document, when the key carries them, which it tells by a length of `indexFields`, the index's own field count, plus two, so that a cursor already moved, which `paginator` returns as a pinned page's `continueCursor`, keeps its length and has only its generation replaced; a cursor whose key's length is neither `indexFields` nor `indexFields` plus two, whose generation is not a number, whose other equality elements differ from the prefix's, or that is not a key, throws a plain `Error` `"The cursor is not from this list"`, after `authorizeQuery` and the read of the active generation and before the list reads a row, so a saved cursor never overrides the query's tenant or status; `null` passes as a start or an end cursor and `undefined` as an end cursor, the validator requiring a start cursor, and `"[]"`, the helper's end-of-list mark, passes as the lawful `endCursor` of the last pinned page and, as a start cursor, reads the list from its first row, which is the helper's behavior and discloses nothing; the read-model library exports the function beside `readModelView` (D9, E-24, Probe 5, Law 11) - indexReadModelList: the index a read-model list reads has `tenantId`, `generation` and the list's equality fields first, then its order field, then `key`, as `by_status` of `tableOrderSummaries` in `spec:application.projection-contract`, so a position within a tenant names at most one row of each generation and a cursor moved into another generation lands between the same rows there (D9, E-9, Law 11) -- fnReadModelView: `readModelView(row: Row): Omit` what a read-model query returns for a row: the tenant, the key, the projection version, the source versions and the projected fields; the built-in page's rows carry `_id` and `_creationTime`, which an object validator of the row's fields refuses, and no document ID or generation number leaves the parent (D8, D2, E-9) +- fnReadModelView: `readModelView(row: Row): Omit` what a read-model query returns for a row: the tenant, the key, the projection version, the source versions and the projected fields; the built-in page's rows carry `_id` and `_creationTime`, which an object validator of the row's fields refuses, and no document ID or generation number leaves the parent in a row; a page's cursors are the position tokens of `fnPageInGeneration`, which carry the generation and, as `paginator` issues them, the row's `_creationTime` and `_id`, and a client passes them back unread, since the list refuses a cursor that is not its own (D8, D2, E-9) - validatorReadModelView: `orderSummaryView = v.object({ tenantId: v.string(), key: v.string(), projectionVersion: v.number(), sourceVersions: v.array(streamVersionValidator), ...orderSummaryFields })` where `orderSummaryFields` are the projected fields `spec:application.projection-contract` declares for `tableOrderSummaries` (E-9, D8) - noActiveGeneration: a query over a read model that has no active generation throws a plain `Error` that names the read model, a technical failure, and never answers an empty page (D9, Law 6) - fnCrossContextView: `export const getOrderWithStock = query({ args: { tenantId: v.string(), orderId: v.string() }, returns: v.union(v.object({ data: orderWithStockDto, sourceVersions: v.array(streamVersionValidator) }), v.null()), handler })` making at most 8 component reads and returning a `VersionedResult` whose `sourceVersions` are the DTOs' stream versions, or reading one cross-context read-model row, which carries `sourceVersions` itself (D8, E-45) From 840b32c31e8e2e32b59930be772577bd367e467f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 06:23:39 +0200 Subject: [PATCH 110/155] design: the close of round 3, unit 3b STATE.md rewritten for unit 3b, the online rebuild of the order summary: the design pass, the chain timing, the three lanes, the integration at 325 of 325, the cutover design, the fold-in's rulings, the review's four passes and sol's three reads, the L2-5 times, the ledger and register rows, and the sweep of the proof records. Tactical decision 41 amended, 48 (tests are the proof) and 49 (the integrator in two stages, the review continued by message) added; the deploy key check in the owner queue; Leads 41. ROADMAP.md: 3b done, the hosted tier next with its prerequisites as they stand. The owner, 2026-10-03: "Tests should be all the proofs we need." and "We need to weed out all examples from the codebase and do all the suggested things." Co-Authored-By: Claude Opus 5.5 --- design/ROADMAP.md | 8 ++-- design/STATE.md | 119 ++++++++++++++++++++++++++++++---------------- 2 files changed, 82 insertions(+), 45 deletions(-) diff --git a/design/ROADMAP.md b/design/ROADMAP.md index b489a94..597cce4 100644 --- a/design/ROADMAP.md +++ b/design/ROADMAP.md @@ -1,6 +1,6 @@ # Roadmap: the next units -Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of the tooling unit on 2026-10-03. +Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of round 3's unit 3b on 2026-10-03. ## How to read it @@ -69,8 +69,8 @@ Done on 2026-10-02 and 2026-10-03, as nine lanes merged into `feature/round-2`, The owner discusses the next session's work before it is prepared. In this order: -1. **3b, the online rebuild of the order summary**, Sc L2-5 at the native tier. It is the next unit, laid out in `layout-3b.md` of the tooling unit. The layout counts thirteen registered functions per composition and 33 proof obligations (1 compiled, 4 pure, 23 `convex-test`, 5 native). It plans three lanes at once (fix, proof, native), then an integrator, a review and a fold-in. Rulings 2 of that unit rules its eight questions. Its Spec source is `design-s4-paper.md` in the round 2 folder: package 3b of section 3 and the rulings of section 7. 3a is merged, so `assertOperator`, `nextTenant` and `gateAllows` exist. The files: `src/read-model/projection.ts` (`projections`, `projectionOf`, the three roles of a written generation, the backfill mode), `src/read-model/tables.ts` (the slimmer `generations` and `generationProgress`, with its field `pass`), `src/read-model/generations.ts` (the roles, and `activateFirstGeneration` removed), a new `src/read-model/rebuild.ts` (one helper per entry, `RebuildConfig`, `batchSizeFor`, `resumeChain`), the order summary's `projections`, the composition's registered rebuild functions and the fixture's read models the same. Every setup that ran the first activation installs through `startGeneration`, its batches and `switchGeneration`, and the first-activation example is renamed `first-rebuild-installs-read-model` with its test in one commit. From section 7: `fillTenants` and its batches with the refusal of `startGeneration` on an empty tenant list beside a grant, `resumeGeneration` with no batch size, the backfill mode's rule for a row that projects to `null`, and the message that sends a started purge to `resumeGeneration`. The proof obligations are at the pure, `convex-test` and native tiers, the native one the L2-5 example: 350 orders in two tenants, batch size 1, live commands in both tenants, an interrupt, a resume, a switch and a rollback. It does not build the paused path, the aggregate rebuild, `@convex-dev/migrations` in any form, an automatic stall check, a purge that starts by itself or the restore's `finishRestoreRun`. -2. **The hosted tier**, a build unit commissioned from `design-hosted-tier.md` of the tooling unit, after 3b. It needs four things first: the deploy key `hosted-ci` and its GitHub secret, both the owner's by hand; the environment `hosted` and its three variables; `patch-hosted-specs.diff` applied; and sitting two's ruling on the words the patch writes. It shows the four hosted examples run by the workflow on `main`: F16's retention, F12's in-place import, the production composition's acceptance and Probe 3's function calls. The local backend stays primary. +1. **3b, the online rebuild of the order summary**, is done, on 2026-10-03: thirteen functions per composition, every read model installed through its first rebuild, the parent list paged across a switch, and Sc L2-5 passing at the native tier on the production composition. `STATE.md`, "Where the work stands", has its numbers: 325 of 325 proof tests at the merge, the L2-5 run's times, the review and three reads of the fold-in. +2. **The hosted tier**, a build unit commissioned from `design-hosted-tier.md` of the tooling unit, is next. Of its four prerequisites, the environment `hosted` with its three variables and the deploy key `hosted-ci` with its secret exist since 2026-10-03; the key check, one command, and the dashboard check are the owner's; `patch-hosted-specs.diff` is applied first in the unit, after sitting two rules the words it writes. It shows the four hosted examples run by the workflow on `main`: F16's retention, F12's in-place import, the production composition's acceptance and Probe 3's function calls. The local backend stays primary. 3. **3d, sitting two.** The language rows OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added (snapshot, tenant fill, the measurement's words) and those of the tooling unit (R5's four, the hosted patch's, 3b's, the operator term); OD-008 with OD-006; OD-138; OD-041, OD-042 and OD-114, the last with Probe 7's facts; OD-044 with F18 against D2; OD-063 with its local numbers; and OD-048, the licence. The F2 comparison is done: every target held on the quiet ThinkPad. 4. **4a, command audit and the diagnostic.** 3a has built `writeAudit` and step 10, so what is left is the Spec text on which outcomes emit a record and where a duration comes from, the sink, and Sc ALL-1 bound. 5. **4c, the history view on paper.** 2c is done, so `OrderCancelled` and `AllocationReleased` exist for its fold. @@ -80,7 +80,7 @@ The owner discusses the next session's work before it is prepared. In this order **3a. The gate and the door.** The gates table, `gateAllows`, `assertWritable` at step 7, close, resume and get for the `all` and tenant scopes, the read of `MAINTENANCE_MODE` before the gate document, and the audit table with `writeAudit` inside every close, resume and abort. At `convex-test`: a command in a closed tenant is refused and nothing is stored, and a close whose audit insert fails leaves the gate open. On the native backend: the environment switch refuses every write, and two writers race a close. -**3b. The online rebuild.** The rebuild's thirteen functions per composition (eight operator entries, four batches and `getGenerations`, as `layout-3b.md` counts them) and the progress row, with the version-selected projection of 2b. A version 2 of the order summary built as generation 2 while `PlaceOrder` runs, a fence bump as the interrupt, verify, switch, intervening writes, rollback, verify again, switch back. Sc L2-5 on the native backend with a record. +**3b. The online rebuild.** The rebuild's thirteen functions per composition (eight operator entries, four batches and `getGenerations`, as `layout-3b.md` counts them) and the progress row, with the version-selected projection of 2b. A version 2 of the order summary built as generation 2 while `PlaceOrder` runs, a fence bump as the interrupt, verify, switch, intervening writes, rollback, verify again, switch back. Sc L2-5 on the native backend with a record. Built on 2026-10-03 as round 3's unit 3b, with generation 2 at version 1 and the two versions on the fixture's `documentTitle` at `convex-test`, as Q1 of its rulings says. **The hosted tier.** Built from `design-hosted-tier.md` section 1, decision 7, in its order: redaction first, then the record shape and the acceptance filter, `hostedTarget`, the selection kinds, the run setup, the local example of the undeclared table, the copy's modules, the four examples and the workflow. It is built when one record carries `target: "hosted deployment"` with no form of the key, the acceptance example passed, and a CI run on `main` says so. Previews, a second deployment, deletion and cost in money are out. The project and the deployment `dev/hosted` exist since 2026-10-03. diff --git a/design/STATE.md b/design/STATE.md index 650aa47..f62163a 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -1,30 +1,30 @@ # State of the design corpus -Written on 2026-10-03 at the close of the tooling unit, which followed round 2 of the roadmap, round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. +Written on 2026-10-03 at the close of unit 3b of round 3 of the roadmap, which followed the tooling unit, round 2, round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. ## Measured at close ``` -232 specs · 5 packs · 130 anchors → 367 nodes · 1222 edges (0 errors, 0 warnings) +232 specs · 5 packs · 131 anchors → 368 nodes · 1223 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings; readiness divergence: [] open questions: 69 Specs, 147 questions, 9 blocking; extensions registered: 53 stated readiness: {'defined': 222, 'idea': 1, 'scoped': 9} -ledger: 278 findings, {'fixed': 148, 'open': 103, 'owner': 1, 'partially-fixed': 26} -open findings by slice: {'L3': 31, 'P': 30, 'S3': 1, 'S4': 19, 'S5': 22} -corpus digest: c4805deb1c987a46 +ledger: 278 findings, {'fixed': 148, 'open': 99, 'owner': 1, 'partially-fixed': 30} +open findings by slice: {'L3': 31, 'P': 30, 'S3': 1, 'S4': 15, 'S5': 22} +corpus digest: 8e9818820eb59f6b protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` -`python3 design/tools/decisions.py`, after the register patch of this close: +`python3 design/tools/decisions.py`, after the register patch of this unit (`314c265`); the close applies none of its own: ``` total: 154 -status: unsorted=0, sorted=100, waiting=22, decided=17, folded=15 +status: unsorted=0, sorted=96, waiting=22, decided=21, folded=15 class: tactical=3, delegated=86, owner=50, null=15 advisor: convex=36, domain=45, operator=34, product=39 ``` -**Round 2 and the tooling unit are on `feature/round-2`, not on `main`.** `main` is at `592f1dc`, the close of round 1. The lanes of round 2 branched from `main` or from `feature/round-2` and were merged into `feature/round-2`, which is pushed; CI passed on its pushes at `1b92bd8` and `a5d339d`. The tooling unit changed no code and no Spec on the branch: its close commits the register, this file and the roadmap. No pull request to `main` is opened until the owner has reviewed the branch. The owner's words: "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue" and "We can discuss details before preparation of work for the next session." Tactical decision 42 holds them. The next lanes branch from `feature/round-2` and merge back into it. +**Round 2, the tooling unit and round 3's unit 3b are on `feature/round-2`, not on `main`.** `main` is at `592f1dc`, the close of round 1. The lanes of round 2 and of 3b branched from `main` or from `feature/round-2` and were merged into `feature/round-2`, which is pushed. CI passed on its pushes at `1b92bd8` and `a5d339d` in round 2, and at `314c265` (7 m 5 s), `2283083` (7 m 22 s) and `de285bb` (7 m 17 s) in 3b. CI runs again on the push of this close. The clean whole-tier run of 3b is CI's: the fold-in's two local whole-tier runs ran while another project's vitest job held the load average at 20 to 24, the first failed two timing-sensitive tests and the second four, and each failed file passed alone. No pull request to `main` is opened until the owner has reviewed the branch. The owner's words: "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue" and "We can discuss details before preparation of work for the next session." Tactical decision 42 holds them. The next lanes branch from `feature/round-2` and merge back into it. The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Each unit works on its own branch, named for the unit, so that `main` stays green. The owner gave the session a mandate on 2026-10-01 to merge at this early stage, and said the same day that a pull request per unit was not a good idea while the design is iterated quickly. So from slice S1 on there is no pull request: CI runs on every push, and the session fast-forwards `main` once CI is green on the last commit. Slice S0 was the one unit merged through a pull request. That held until the merge of 2026-10-02. @@ -34,7 +34,36 @@ From this merge on, a unit reached `main` through a pull request, tactical decis ## Where the work stands -Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, round 1 of that roadmap is on `main`, round 2 is on `feature/round-2`, and a design unit for the tooling followed on the same branch. +Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, round 1 of that roadmap is on `main`, round 2 is on `feature/round-2`, a design unit for the tooling followed on the same branch, and so did round 3's first unit, 3b, the online rebuild of the order summary. + +Round 3, unit 3b, `application-platform-round-3`, on 2026-10-03 from about 03:32 to 07:00, on the ThinkPad, from `feature/round-2` at `62b8817` to `399b410`. The owner's words at the opening: "please plan and execute the scope, including validaiton of the hosted convex env. use fable 5.1 for designs of bigger pieces of solution, opus, astra and sol for execution." The hosted check at the opening was read only: the environment `hosted` shows its three variables and its secret, and `convex env list` with the owner's login reached `wandering-mule-403`; whether the key itself authenticates is the owner's one command (Owner queue). During the unit, at the integration, the owner asked whether the proof records were needed: "Tests should be all the proofs we need." and "We need to weed out all examples from the codebase and do all the suggested things." Decision 48 holds it. The main thread wrote two rulings pages, `rulings-1-round-3.md` and `rulings-2-on-the-integration.md`, and `answers-lanes.md` keeps every answer with the ruling after it. What each lane showed, at the tier named: + +- The design pass. One Fable agent in 10 minutes, a determinacy pass over 3b's Spec text: 29 sentences in five Specs and the renamed example, two patches proven on a scratch copy. They pin `documentTitle`'s two versions, the write rule's reading of "older", `updatedAt` on every progress-row patch, the exports beside the helpers, each entry's and each pass end's row patch, and the L2-5 run's order. The main thread changed one sentence: the first tenant's order cancelled during the interrupt is one whose row generation 2 already holds, read from the table, and not the one with the lowest order ID. Committed as `2b6b9db` before the lanes branched. +- The chain timing. An Opus agent on the quiet machine (load 0.62 to 0.73), native tier, three runs of a 100-link chain of `runAfter(0)` mutations: 497.5, 442.7 and 452.9 ms, a mean of 4.69 ms a link, the longest gap 7.99 ms, a median of about 2.4 ms of backend execution per link. So the chaining of the L2-5 run's about 780 links costs about 3.7 s, and the 300 s timeout and the 30 s stall bound stood. +- The three lanes, at once, each an Opus lead with one GPT job, and nothing went back to any job. Fix: `gpt-6.1-sol` at medium built package 3b in 7 min 55 s; the lead fixed five comments and listed five text items. Proof: `gpt-6-astra` at high wrote 325 tests in 12 min 43 s (28 compiled, 95 pure, 202 `convex-test`), every one of the 28 obligations of its tiers covered and all red before the fix; the lead added two and listed six items. Native: astra at high wrote the B4 setups, the renamed example and the L2-5 test in 15 min 28 s, red only at the missing fix; the lead made five small fixes and listed five items. The main thread ruled the sixteen items on receipt. +- The integration. An Opus integrator merged fix, proof and native with no conflict, codegen unchanged after each, in two stages while the native lane still ran. At the merge 325 of 325 proof tests passed after 19 wrong type tests were corrected (18 a misplaced `@ts-expect-error`, one the `| undefined` of `exactOptionalPropertyTypes`); the fix was wrong nowhere. It corrected three setups no lane rewrote, the L2-5 example's opening line, and seven native tests that counted the install's two batch rows. At `a013ba4`: `npm test` 943 passed; the native tier 153 tests, 128 passed, 20 failed, 5 skipped; acceptance 30 passed, 2 failed (Sc L2-5, Sc L2-9), 4 missing. The 20 were four questions for the main thread. +- The cutover design. The L2-5 run's subscription threw `InvalidCursor` at the switch, because the built-in `paginate` refuses a cursor from another query and the active generation is in the range. One Fable designer in 7 minutes, with a spike at `convex-test`, showed the raw cursor of `paginator` failing three ways: a page-two cursor answered page one again, a pinned page threw `ConcatStreams in wrong order`, and after a switch back a cursor answered the retired generation's rows. The ruling: a parent list pages with `paginator` as a context list does, `by_status` ends with `key`, and `pageInGeneration` moves each cursor of the client's pair into the active generation. +- The fold-in, one Opus agent in three passes, from rulings F1 to F8 and G1 to G5, then H1: + - F1: the interrupt and a running batch both patch the progress row and one reruns, as designed; the L2-5 sentence now says no live command or parent query names `generationProgress` in its `occInfo`, and the entries' and batches' reruns there are counted. + - F2: the cutover design applied in both compositions' lists and `src/read-model/queries.ts`, with the spike's four cases as `convex-test` cases. + - F3: the slimmer row's observations rerun alone: 95 bytes fewer read per generation row read (190 with a building generation), 9 bytes more written per summary row because `by_status` ends with `key`. + - F4: the left-behind schema check counts `generationProgress`, `tenantFill` and `projectionMarkers`, and Sc L2-9 passes. + - F5: the two proof tests a lawful build failed: nothing of `ctx` read before `assertOperator`, and the write-rule fake answering `.first()` and `.unique()` alike. + - F6: seven text rulings of the lanes, among them a batch with no progress row throwing `"No progress for generation " + generationId` and `getGenerations` of an undeclared read model answering `[]`. + - F7: five ledger rows `partially-fixed`; the register patch written for the main thread, because the agent's classifier refused `decisions.py apply`. + - F8: the gates, below. + - G1: `rollbackGeneration` refuses while the tenant fill is in its pass `fill`, Spec, code and one `convex-test` case. + - G2: the fill copies tenant IDs as the grants hold them, one sentence, no code. + - G3: the install's one batch, its rows and the stated version asserted; a per-entity read model writes no marker, so the test asserts none. + - G4: the L2-5 test counts the backfill and verify completions that wrote against `batchesDone`. + - G5: the three new tables in every stored-row comparison. + - H1: a pinned page's `continueCursor` is already moved, so `pageInGeneration` drops `_creationTime` and `_id` only from a key of the index's length plus two. + At `8a6af06`: `npm test` 951 passed. Its two whole-tier runs ran under the outside load: 1,096 passed and 2 failed at load 1.97, then 1,094 passed and 4 failed at load 5.94, with Sc L2-5 passing in both; every failed file passed alone. +- The review, `gpt-6-astra` at xhigh under an Opus lead, started on frozen `a013ba4` so that it overlapped the fold-in. First pass, 13 min 10 s: the native tier 127 passed and 21 failed (F1 to F4 and a Probe 4 timeout under load), 25 mutations with 18 caught, two text gaps (G1, G2). Its later passes, the same thread sent each head, at `314c265`, `2283083` and `de285bb`, 1 h 45 m in all over four runs: at `314c265` it raised the two findings sol's first read raised (a saved cursor's tenant read through another tenant, a partial cursor read a retired generation), at `2283083` sol's codec pair, and all four are closed at `de285bb` in both compositions, at `convex-test` and natively. Of 43 mutations 39 were caught: M17 and M20 to M23 turned caught with G3 and the fold-ins, and every cursor and codec mutation, M26 to M43, was caught; M06, M25, M31 and M32 survive (Leads 41). Eight of nine L2-5 replays were caught, and R2 survives as G4 allows. The native tier in its copy at `de285bb`: 103 files, 146 passed, 2 failed, 5 skipped, the whole run 1,102 passed, 2 failed and 5 skipped in 274.9 s, both failures Probe 4 backend timeouts that passed alone; at `314c265` 144 passed and 4 failed, each passing alone. Its one text gap, `read-models.sdp.md:99`, the main thread fixed in `399b410`: no document ID or generation number leaves the parent in a row, and a client passes a page's cursors back unread. +- Three reads of the fold-in by one `gpt-6.1-sol` job at medium, each on a fresh frozen copy. At `314c265`: four findings, among them a saved cursor of another tenant that overrode the authorized tenant and a six-element cursor that read generation 2 while generation 1 was active; five tests that passed with their rule removed; every recomputed number equal (the L2-5 run's checkpoints 395 + 525 = 920 against 626 subjects, 3 registry reruns and 1 progress-row rerun against a bound of 39); seven text gaps. Ruled K1 to K4: the cursor's equality prefix compared and anything else refused with `"The cursor is not from this list"`, `getGenerations` over the configured read models, the page tests through the entries, the seam's qualification. At `2283083`: two findings with one root, `paginator`'s own codec, so a tenant `t-undefined` was refused its own continuation and `_t-undefined` read its rows. Ruled L1 to L4: the key decoded and encoded as `paginator` does, held to the helper by test. At `de285bb`: no finding on a healthy or ordinary failure path, every number equal, and four codec mutations and four text gaps left (Leads 41). The unit stopped there on decision 19. Sol's job took 41.9 minutes over four turns, a token refresh cutting the first. +- The L2-5 file, native tier, production composition: 39.0 s alone at `a013ba4` (load 2.08), 72.1 s in the integrator's whole-tier run (load 1.14) and 106.6 s in its first (load 3.41); 40.6 s alone at the fold-in (load 3.97), 94.3 s and 287 s in the fold-in's two whole-tier runs under the outside load; 43.5 s and 42.3 s alone after K and L. Every time is well inside the 300 s timeout; none is a quiet-machine timing. +- The ledger and the register. Five ledger rows are `partially-fixed` with their reasons: the checkpoint on the row commands read, the tenant list, the batches bounded in documents only, and the layout's two rows of a native rollout to version 2 and a native absence, which stay with the fixture at `convex-test` by Q1. S4's open findings went from 19 to 15. OD-057 (convex), OD-071, OD-075 and OD-080 (domain) are `decided` by their advisors' checked leans, applied by the tool in `314c265`; OD-149 stays `sorted`, its lean unchecked. +- The sweep. The owner's words at about 04:45, after seeing records still written: "I can see we are still practicing proof records. [...] Is this needed? Tests should be all the proofs we need. If we decide to remove this, how can we actually make it happen this time?" and "We need to weed out all examples from the codebase and do all the suggested things." An Opus agent in 3.4 minutes took the pattern out of `AGENTS.md`, `SESSIONS.md` and `design/advisors/README.md` (`e377f58`) and out of the guide and the project notes. No record file was written after it. The tooling unit, `application-platform-tooling-1`, on 2026-10-03 from about 01:50 to 03:00, on the ThinkPad, from `feature/round-2` at `8cc40e8`. The owner's words at the opening: "we will document and start designign the backed tooling updates, not do any changes in the next session. but record data for sure." So no code and no Spec changed on the branch. Each proposed Spec change is a patch in the unit folder, proven on a scratch copy and not applied, and the close commits the register, this file and the roadmap. No native backend started and no hosted deployment ran anything. The main thread wrote two rulings pages, `rulings-1-tooling-1.md` and `rulings-2-on-the-advisor.md`, and ruled on the leads' answers. What each lane showed, at the tier named: @@ -137,13 +166,13 @@ What passed, and at which tier, with backend release `precompiled-2026-09-28-5c7 Every Layer 1 scenario runs on the fixture composition. The Layer 2 scenarios run on the production composition, except the one that needs a fault. -What the code has and what it does not have yet, after round 2. The Specs pin all of it and say nothing about which part is built: the owner's direction of 2026-10-01 is that a Spec carries no temporal information. +What the code has and what it does not have yet, after round 3's unit 3b. The Specs pin all of it and say nothing about which part is built: the owner's direction of 2026-10-01 is that a Spec carries no temporal information. -- Pipeline: both entries, steps 1 to 6 with the ID length bound, the gate read of step 7 through `assertWritable` before the `OperationRef` is minted, step 8, step 9 with the `writes` check, the row budget and the row count, the receipt insert of step 10 with the audit record of a declaration that sets `audit`, and the return of step 11. Not yet: the diagnostic. +- Pipeline: both entries, steps 1 to 6 with the ID length bound, the gate read of step 7 through `assertWritable` before the `OperationRef` is minted, step 8, step 9 with the `writes` check, each writable generation written with the projection its row's version selects, the row budget and the row count, the receipt insert of step 10 with the audit record of a declaration that sets `audit`, and the return of step 11. Not yet: the diagnostic. - Declaration: `writes` is required, `readModels` and `audit` are optional. No `irreversible` field yet, and `retention.afterExpiry` is `"delete"` only. There is no `commandRegistry` and no registration check: the Specs dropped them, and the compiler over the generated api is the check of Sc L2-4. - The gate and the audit records: `src/gate` and `src/audit`. The `maintenanceGates`, `operatorAudit` and `auditRecords` tables, `gateAllows` reading `MAINTENANCE_MODE` before the gate document, `assertWritable`, `scopesOfUseCase`, `closeScope` and `resumeScope` with their audit record in the same mutation, `writeAudit` and `writeOperatorAudit` with no catch, and the four operator entries `closeGate`, `resumeGate`, `getGate` and `getGateAudit` that both compositions re-export. Not yet: a source scope closed by `startGeneration`, the restore's entries, the audit queries by tenant, a typed `env`. -- Authority: `establishActor`, `authorize`, the grants table, the `tenants` table, and `insertGrant`, which inserts a tenant's row with its first grant, `revokeGrant` and `nextTenant`; both grant helpers read the gate first. `assertOperator` checks the stated operator of an operator entry and returns its trimmed text. The library registers no grant function; each composition registers its own two internal grant mutations. Not yet: `fillTenants` and its batches, which the rebuild Spec now pins. -- Read models: the per-entity form only, written from the DTOs of a command's stream entries, with one `projection` on the declaration. The code still installs a read model by the first activation, while the Specs now install it through its first rebuild; the first-rebuild example is bound to its steps and skipped. Not yet: the rebuild (`projections` by version, `generationProgress`, `startGeneration`, the backfill and verify batches, switch, interrupt, resume, rollback, abort and purge), `applyAggregate`, `applyHistoryProjection`, `projectionMarkers`, `getGenerations`. +- Authority: `establishActor`, `authorize`, the grants table, the `tenants` table, and `insertGrant`, which inserts a tenant's row with its first grant, `revokeGrant` and `nextTenant`; both grant helpers read the gate first. `assertOperator` checks the stated operator of an operator entry and returns its trimmed text. The library registers no grant function; each composition registers its own two internal grant mutations. `fillTenants` and `fillTenantsBatch` fill the tenant list from the grants, with their progress on the `tenantFill` row. +- Read models: the per-entity form only, written from the DTOs of a command's stream entries. A declaration carries `projections` by version, latest first, and `projectionOf` picks the one a generation's row records. The slimmer `generations` row, `generationProgress` with its `pass`, the `projectionMarkers` table, the three roles of a written generation, and the backfill mode with its `null` rule. The rebuild is built: `src/read-model/rebuild.ts` with one helper per entry over `RebuildConfig`, `batchSizeFor` and `resumeChain`, and in each composition a `rebuild` module of thirteen functions (`startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration`, `purgeGeneration`, `fillTenants`, the backfill, verify, purge and fill batches, `getGenerations`). The first activation is gone: every read model, in every setup, is installed through its first rebuild, and the renamed example `first-rebuild-installs-read-model` runs on the fixture. A parent read-model list pages with `paginator` and moves the client's cursors into the active generation with `pageInGeneration`, refusing a cursor of another list. Not yet: the paused path (`pauseRequired`, source scopes, history batches), the aggregate rebuild and `applyAggregate`, `applyHistoryProjection`, a stall check, a purge that starts by itself. - Receipts: the key, the fingerprint, lookup, classification and insert. An expired receipt is treated as absent and deleted in the same mutation. Not yet: tombstones and the sweep. - Outcome boundary: the wire shapes, `reject`, `refuseTransient`, `normalizeThrown` with the closed code list, `classifyThrown`. Not yet: the conditional dispatcher and `carryOrReject`, which has no caller. - Context: operations, `get` and `list` with `includeDeleted` for a trusted caller. Not yet: `history`, `rebuild`, `byOperation` and `maintenance.writeBaseline`. @@ -151,7 +180,7 @@ What the code has and what it does not have yet, after round 2. The Specs pin al - Journal and adapter: the single mapping only. Not yet: baselines and migrations (step 3a, `migrateToSchemaVersion`, `defineMaintenance`, the driver), `rebuildStream`, the derived mapping and `streamParts`. A stream row behind the registration's `stateSchemaVersion` is a plain error for now. - The example domain: `PlaceOrder`, `ReceiveStock` and `CancelOrder`, the order's `cancel` and the stock item's `release`, `OrderCancelled` and `AllocationReleased`, with a stock item ID bound of 64 bytes on `PlaceOrder`'s lines and `ReceiveStock`'s items. `getOrderWithStock` and the parent queries over Inventory are not built, because no Spec pins their shapes; `inventory.read` is seeded and nothing asks for it yet. `installedLayers` of the production composition is `[]`, because the Spec is silent. - Harness: admin access reads `_scheduled_functions`, exports and imports a snapshot, deploys a temporary copy of a composition with `deployTemporary` and hands back a completion's log lines; a ZIP reader in Node reads the export. -- Not yet, beyond the rebuild: the history view (a unit of round 5 by F10, its rules first in 4c), baselines and migration, the restore procedure, the receipts sweep and the diagnostic. +- Not yet, beyond the per-entity rebuild: the history view (a unit of round 5 by F10, its rules first in 4c), baselines and migration, the restore procedure with its `finishRestoreRun`, the receipts sweep and the diagnostic. What changed in the Specs during S1, and why: @@ -182,37 +211,28 @@ Review round 3 stands as it was for the rest of the corpus: no lens approved. Th | Slice | Builds | Open findings | Blocker, major, minor | |---|---|---|---| | S3 | Measurement: 1, 10 and 100 lines, without and with contention. Repeated after S4 and S5 | 1 | 0, 0, 1 | -| S4 | Generations and rebuild, the history view and the write pause, with the tenant list, operator entry points and gate audit the rebuild uses. The paper, the gate and Probe 6 are done | 19 | 0, 10, 9 | +| S4 | Generations and rebuild, the history view and the write pause, with the tenant list, operator entry points and gate audit the rebuild uses. The paper, the gate, Probe 6 and the per-entity online rebuild are done | 15 | 0, 6, 9 | | S5 | Baseline migration, restore, the rest of operators and audit, diagnostics, the receipts sweep, maintenance jobs. Probe 7's local part is done | 22 | 0, 12, 10 | | L3 | Layers 3 to 6, written from what the experiment shows | 31 | 0, 12, 19 | | P | Polish with no effect on what is built: citations, markers, stale sentences, README | 30 | 0, 0, 30 | -Twenty-six more findings are `partially-fixed`, each waiting for its slice. Remove an unsupported generic promise before designing a helper no view needs. +Thirty more findings are `partially-fixed`, each waiting for its slice. Remove an unsupported generic promise before designing a helper no view needs. ## Next unit -`ROADMAP.md` is the order, ruled as cut by decision 36, and rounds 1 and 2 are done. The next unit is the online rebuild of the order summary, roadmap 3b, with Sc L2-5 at the native tier, as `layout-3b.md` in the tooling unit's folder lays it out. That means thirteen functions per composition, 33 proof obligations, and three lanes at once (fix, proof, native), then an integrator, a review and a fold-in. Package 3b and section 7 of `design-s4-paper.md` in the round 2 folder remain its Spec source. Its eight questions are ruled as rulings 2 of the tooling unit says: +`ROADMAP.md` is the order, ruled as cut by decision 36. Rounds 1 and 2 are done, and so is 3b, the first item of round 3. The next unit is the hosted tier, round 3's item 2, a build unit commissioned from `design-hosted-tier.md` in the tooling unit's folder: its build cut, its order and its "built when" list. The owner's words of 2026-10-03: "Local backend should be primary, IMO. But happy to take the hosted as default if easier for start." So the local backend stays the primary target, and the hosted deployment is a target of the native tier on a schedule, never in `test:all`. Before it starts: -- Version 2 goes on the fixture's `documentTitle`, at `convex-test`. -- The renamed example goes on the fixture, with four of its lines changed. -- The journal check is by operation. -- A `CancelOrder` runs on one of the second tenant's 100 seeded orders while the chain is interrupted, and asserts the order's final row `cancelled`. -- The two ledger rows stay `open`, and the fold-in proposes `partially-fixed`. -- The ledger's extra native shares run at `convex-test` only. -- The L2-5 file sets 300 s, beside the test's own 30 s stall bound. -- One 100-batch chain is timed on the quiet machine before the lanes start. - -The five register rows are recorded with new leans, four of them checked (Owner queue). The setup change covers the eight helpers and the test files that call `activate` directly. The factory is not tried. The brief is written at the next session from `layout-3b.md` and rulings 2; the owner's discussion of the layout happened in this unit and ended in "I agree with all recommendations". The main thread ruled at this close that 3b may start: decision 2 asks for no open blocker inside a slice, and the five rows are recorded with leans that keep the Spec text as written, four of them checked, so the brief builds the same text whether or not they are `decided`; they become `decided` when 3b's fold-in applies them, as `SESSIONS.md`, "Who decides", has a `delegated` row ruled by its checked lean. The layout's lane table still says "16 test files" for the native lane's setups, which rulings 2 corrects. - -After 3b, the hosted tier's build, as a unit commissioned from `design-hosted-tier.md` in the tooling unit's folder: its build cut, its order and its "built when" list. The owner's words of 2026-10-03: "Local backend should be primary, IMO. But happy to take the hosted as default if easier for start." So the local backend stays the primary target, and the hosted deployment is a target of the native tier on a schedule, never in `test:all`. Before it starts: - -- Done by the owner at the owner's own prompt on 2026-10-03, after the close commit: the GitHub environment `hosted` with its `main` branch policy and the three variables (`HOSTED_DEPLOYMENT_URL` as the printed URL with the region, `HOSTED_DEPLOY_KEY_NAME` as `hosted-ci`, `HOSTED_PLAN` as `Professional`), the deploy key `hosted-ci` made in the dashboard on `wandering-mule-403` (its value begins `dev:wandering-mule-403|`, the owner's reading), and the secret `HOSTED_DEPLOY_KEY` in that environment. `gh secret list --env hosted` and `gh variable list --env hosted` showed them. +- Done: the setup, which made the project `libar-platform-hosted` and its development deployment `dev/hosted` (`wandering-mule-403`). +- Done by the owner at the owner's own prompt on 2026-10-03: the GitHub environment `hosted` with its `main` branch policy and the three variables (`HOSTED_DEPLOYMENT_URL` as the printed URL with the region, `HOSTED_DEPLOY_KEY_NAME` as `hosted-ci`, `HOSTED_PLAN` as `Professional`), the deploy key `hosted-ci` made in the dashboard on `wandering-mule-403` (its value begins `dev:wandering-mule-403|`, the owner's reading), and the secret `HOSTED_DEPLOY_KEY` in that environment. `gh secret list --env hosted` and `gh variable list --env hosted` showed them. +- Still the owner's: the key check, one command in a shell that holds the key (Owner queue), which shows that `hosted-ci` authenticates against `wandering-mule-403`. - Still the owner's: the dashboard check. The project showed a second development deployment, `clean-capybara-16`, which no agent created and which is most likely the owner's personal dev deployment from the dashboard's first open; if nothing uses it, the owner deletes it so that the project holds `dev/hosted` only, no production deployment and one key. -- `patch-hosted-specs.diff` is applied after sitting two rules its words. +- First in the unit: `patch-hosted-specs.diff` applied, after sitting two rules its words. Its first runs are the owner's from this machine. The four hosted examples run by the workflow once it is on `main`, because GitHub runs a schedule from the default branch. -`slice` S3 is round 1's attribution and round 2's measurement, the quiet timings included. S4 is round 2's paper and gate, and the builds of rounds 3 and 4. S5 is round 5. The paper decisions S4's rebuild needs are taken; those of the history view and of S5 are listed below. +Then sitting two, roadmap 3d, as the Owner queue lists it. Its ruling on the hosted patch's words comes before that patch is applied, so the owner may take it first. + +`slice` S3 is round 1's attribution and round 2's measurement, the quiet timings included. S4 is round 2's paper and gate, round 3's online rebuild, and the builds of round 4. S5 is round 5. The paper decisions S4's rebuild needs are taken; those of the history view and of S5 are listed below. How S2 was run, for the next slice. Tactical decisions 25 to 27 hold the rules. Two Fable designers with disjoint subjects took about twenty minutes. One workflow run of thirteen Opus agents then wrote the Specs, built four packages in sequence, bound the scenarios in four worktrees and integrated, in about an hour and a half. The main thread stopped and resumed that run once, at a seam between two packages, to add corrections: finished agents came back from the cache. The scripts, the rulings and the reports are in the S2 folder the project notes name. @@ -221,7 +241,7 @@ What S2 left, as it stands now: - The measured read count of `PlaceOrder`, 3N + 5, has every term attributed (round 1) and stays the cost (round 2, C3). - Sc L2-9's native observation of the deployed function list stays deferred: OD-021's lean takes the committed generated api as the pinned list. Its concurrency half is bound, the contention run. - A binding admits only the per-entity read model. The build that takes Sc L2-6 says how a history and an aggregate read model are bound and installed. -- The first activation, which the code still runs, claims nothing about source events that already exist. The Specs replace it with the first rebuild, which 3b builds. +- The first activation is gone: 3b installs every read model through its first rebuild, which reads the source events that already exist. Decisions each later slice needs on paper before its code: @@ -245,7 +265,7 @@ What the platform review of 2026-10-02 asks of the later units. The ledger owns ## Tactical decisions taken on 2026-10-01 -The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words, and 46 and 47 during the tooling unit on 2026-10-03. Any of them can be reopened. +The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words, 46 and 47 during the tooling unit on 2026-10-03, and 48 and 49 during round 3's unit 3b the same day. Any of them can be reopened. 1. Slices replace the eight fix units of the previous plan. 2. A slice is built on Specs at `defined` with no open blocker inside the slice. The owner states `ready` after the slice passes. @@ -287,13 +307,15 @@ The owner accepted the first nine on the session's recommendation. The session t 38. A round runs as one long orchestration, the way S1 and S2 ran and the guide's "Speed with assurance" says: one page of rulings in every brief, one Workflow run of Claude agents with GPT jobs started beside it, packages delegated and never findings, astra on the proof and the measurement, sol on bookkeeping and bounded builds, one integrator, one review behind the write lane, one pull request per lane with significant scope. The owner's words on 2026-10-02: "i would like to do a long orchestration based on notes made by fable orchestrating claude and gpt models. the approach enable significant scope in PRs." 39. A lane's worktree. Each lane of a round is a git worktree under `~/dev-libar/wt/` on its own branch from `main`, with `node_modules` a link to the main tree's and the backend binary named by `CONVEX_BACKEND_BINARY`, and no `.cache` link, because `.gitignore` ignores the directory and not a link to it. Every brief says: never run `npm ci` or `npm install` in a worktree. A link is never created over an existing link. The main thread merges `main` into a lane by hand where two lanes touched one line, and runs the gates before the pull request. 40. The owner delegated the forks of sitting one to the main thread on 2026-10-02 ("Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions.") and, when the register tool refused a session ruling on an `owner` row, chose the cleanest path: "i agree with the least noisy and cleanest path forward." So the twelve rows are recorded as the owner's, each ruling ending with both sentences, the tool is unchanged, and OD-048 stays open. -41. A round's lane is owned by an Opus lane lead from the brief to the last commit. It writes the GPT job's brief, starts the job with the companion from Bash, reads every line the job wrote, fixes what sits below the line itself, sends one numbered list of what sits above it back to the same job, and answers the main thread in fifteen lines. The main thread rules on the answers and merges. Nine lanes ran so in round 2; the project notes hold the numbers. +41. A round's lane is owned by an Opus lane lead from the brief to the last commit. It writes the GPT job's brief, starts the job with the companion from Bash, reads every line the job wrote, fixes what sits below the line itself, sends one numbered list of what sits above it back to the same job, and answers the main thread in fifteen lines. Since decision 48 the lead writes no record: the answer is the record, and the job's report stays in its job directory. The main thread rules on the answers and merges. Nine lanes ran so in round 2 and three in 3b; the project notes hold the numbers. 42. The owner's words of 2026-10-02 and 2026-10-03: "we will need a feature branch" and "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue". So a lane branches from `feature/round-2` and merges back into it, and nothing goes to `main` until the owner has reviewed the branch. Decisions 32 and 39 hold for the rest: a review before `main`, and a worktree per lane. And: "We can discuss details before preparation of work for the next session." 43. One Fable designer took the whole S4 paper, both subjects, because the gate, the rebuild and the operator entries share surfaces, and the owner said "fable only for key design decisions on a larger bit of roadmap". Opus leads, sol and astra did the rest. Fable was spent on the main thread and that one designer. 44. The ThinkPad (Fedora, an Intel i5 with 16 GB) is the development machine for now. The owner's words of 2026-10-03: "We will continue development of this project here on the thinkpad. It is not ideal, but sufficient. M3 Mac is continuting with macos native development." Seven lanes at once drove its load average to 13 to 18 and made native tests time out. So at most three or four lanes that start native backends run at once, a test that fails on a timeout is rerun once alone, and a latency or throughput timing is taken while nothing else runs. 45. An integrator never uses `pkill` on vitest: `pkill -f "vitest run"` matched every lane's run once and interrupted another lane's native file. A run is stopped by its own process ID. 46. When the owner asks for one, a design unit's key decisions go to one Fable advisor, with the owner's principles named in its task: quality, simple design, a maintainable platform. The owner's words of 2026-10-03: "please spawn fable advisor to recommend decisions. let's keep it less noisy and use one advisor. quality, simple design, maitanable platform are guiding principles." An Opus lead turns the memo into the design file and the Spec patch. Sol checks both, and the memo before them, each on a copy with one planted citation. An Opus draft written before the Fable spend was confirmed, such as the hosted lane's `draft-hosted-opus*`, is input, not a decision. With no such request, the standing word holds: "no advisors now". 47. No agent of this harness writes a secret. On 2026-10-03 the main thread's harness refused a secret-store write even through an agent, a "Secret-Store Writes" denial, and the main thread does not work around it. The owner creates a deploy key by hand and stores it at an interactive prompt, never in a chat or in a file in the repository. The setup around the key (the project, the deployment, the GitHub environment and its variables) is an agent's work. The owner's words: "This setup cannot be documented and executed by an agent? With gh cli, convex cli, and other tooling?" and "1. and if someting is an issue, i will jump in." When an agent's own permission check refuses a step, as it refused the setup lane's `gh api` write, the agent stops and hands the owner the exact commands. +48. Tests are the proof. The owner's words of 2026-10-03, during 3b: "Tests should be all the proofs we need." and "We need to weed out all examples from the codebase and do all the suggested things." What stays: one rulings page per unit, an answer of at most fifteen lines from each agent, one run-log line per GPT job in the project notes, this file, a design note when a Fable agent decides, the ledger and the register, and the harness's run records under `evidence/runs/`, which the acceptance check reads. What goes: `record-*.md` files, saved copies of reports and gate logs, a command per number, and a test shown to fail under a mutation outside the review lane; mutation testing is the reviewer's, in its own report. The repository's gates, which CI runs on every push, are the proof, so a fold-in runs the files it touches alone and CI's run on the push is the whole tier. It is written in `AGENTS.md`, `SESSIONS.md` and the orchestration guide, commit `e377f58` and its twin in the context repository. +49. The order of the work after the lanes. The integrator starts in two stages while a lane still runs: it merges the lanes that are done, rules their failures, and takes the last lane's head by message. The review starts on the merged head before the fold-in, so that its mutations and its native runs overlap the fold-in, and the same job is sent each fold-in head by message and reads the diff from the merged head to it. ## Owner queue @@ -301,12 +323,21 @@ Every open decision is a row of `design/decisions/register.json`, and `python3 d ``` total: 154 -status: unsorted=0, sorted=100, waiting=22, decided=17, folded=15 +status: unsorted=0, sorted=96, waiting=22, decided=21, folded=15 class: tactical=3, delegated=86, owner=50, null=15 advisor: convex=36, domain=45, operator=34, product=39 ``` -Decided since the last close: OD-010, installation through the first rebuild, and OD-014, the projection chosen by the generation's version. Both were decided by the domain advisor with their checked leans of 2026-10-02 as the ruling, as rulings 2 of the tooling unit says. The advisor's patch (`patch-register-3b.json`) went in through `decisions.py apply`. OD-057 and OD-080 moved from `waiting` to `sorted`, and OD-057, OD-071, OD-075, OD-080 and OD-149 carry the advisor's new leans. A second patch through the tool set `checkedBy` to `gpt-6.1-sol, medium, 2026-10-03: every checked claim held` on the first four, from sol's recheck. OD-149's stays null: its `ifWrong` says a tenant missing from the list gets no row until `fillTenants` and a rebuild, while a live write in the active generation inserts a missing row (`projection-contract.sdp.md:49`). That changes no `do`, but the register's form for `checkedBy` is "every checked claim held". The hosted patch, when it is applied, adds OD-155, the hosted tier's money, as an `owner` row. +The owner's one command before the hosted tier: the check that the deploy key `hosted-ci` authenticates against `wandering-mule-403`. It needs the key in the owner's shell, so it is the owner's, in an empty directory and read only. Paste the key at the silent prompt: + +``` +cd "$(mktemp -d)" && read -rs CONVEX_DEPLOY_KEY && export CONVEX_DEPLOY_KEY +HOME=$(mktemp -d) ~/dev-libar/libar-platform/node_modules/.bin/convex env list --names-only; unset CONVEX_DEPLOY_KEY +``` + +An empty list with no error means the key authenticates. The dashboard check is the owner's too ("Next unit"). + +Decided since the last close: OD-057 by the convex advisor, and OD-071, OD-075 and OD-080 by the domain advisor, each with its lean checked in the tooling unit as the ruling, which the owner agreed to ("I agree with all recommendations"). The fold-in of 3b wrote the patch, the agent's classifier refused `decisions.py apply`, and the main thread applied it through the tool with the decider corrected to each row's advisor, `314c265`. At the close before, OD-010 and OD-014 were decided by the domain advisor, and OD-057, OD-071, OD-075, OD-080 and OD-149 took the advisor's new leans, the first four with `checkedBy` from sol's recheck. OD-149 stays `sorted` and its `checkedBy` null: its `ifWrong` says a tenant missing from the list gets no row until `fillTenants` and a rebuild, while a live write in the active generation inserts a missing row (`projection-contract.sdp.md:49`). That changes no `do`, but the register's form for `checkedBy` is "every checked claim held". The hosted patch, when it is applied, adds OD-155, the hosted tier's money, as an `owner` row. At the close before, sitting one's twelve `owner` rows were recorded as the owner's (decision 40), and OD-007 was decided by the convex advisor. OD-154, the grant read, is a `delegated` row with its measured cost. @@ -325,7 +356,7 @@ Sitting two waits for the owner. Its forks are drafted the same way as sitting o `delegated` rows that wait for the next sort of their lens, each with the round's result on its lean: OD-066 (3N + 5 with every term attributed; the journal tail read is 10 documents and 6,010 bytes, 29 percent of the documents read on ten lines), OD-120 (Probe 7's states) and OD-154 (3 documents and 918 bytes for the grant read) for the convex lens, and OD-081 (Probe 7's restore answers) for the operator lens. OD-057 has left this list: its new lean closes on Probe 6's result and is checked. The session may not decide a `delegated` row (`decisions.py`: "session may decide only tactical rows"). -The 86 `delegated` rows carry the leans of the first panel run, five of them replaced in the tooling unit. A `gpt-6.1-sol` check read the first 23: twelve held on every checked claim and carry `checkedBy`; eight hold with a gap and three do not hold (OD-066, OD-103, OD-109), each waiting for its advisor's correction with the check's findings in the unit folder the project notes name. The tooling unit's recheck adds four rows that carry `checkedBy` (OD-057, OD-071, OD-075, OD-080), so sixteen carry it, OD-010 and OD-014 among them, now decided. The other leans are unchecked. A checked lean is applied to the Specs as a ruling by a later unit, one Spec family at a time, and the row becomes `decided`. The rest carry a one-line lean or wait for a probe, a measurement or a build (`--waiting`). +The 86 `delegated` rows carry the leans of the first panel run, five of them replaced in the tooling unit. A `gpt-6.1-sol` check read the first 23: twelve held on every checked claim and carry `checkedBy`; eight hold with a gap and three do not hold (OD-066, OD-103, OD-109), each waiting for its advisor's correction with the check's findings in the unit folder the project notes name. The tooling unit's recheck added four rows that carry `checkedBy` (OD-057, OD-071, OD-075, OD-080), so sixteen carry it; six of them are now decided, OD-010 and OD-014 at the tooling close and those four in 3b. The other leans are unchecked. A checked lean is applied to the Specs as a ruling by a later unit, one Spec family at a time, and the row becomes `decided`. The rest carry a one-line lean or wait for a probe, a measurement or a build (`--waiting`). ## Leads @@ -385,7 +416,7 @@ A session confirms or drops each. None is a finding yet. - A native run over history of the maximum event size waits for the unit that builds rebuild. One attempt hit the fixture stream's own state budget first. 24. A Claude agent in this harness may not write a file it calls a report. It may write a design file, a patch and a spike. The main thread saves a hand-back by extracting it from the agent's transcript with a script. 25. From the platform review, not findings: - - Pagination across a generation switch. Keep two subscribed pages pinned with both cursor endpoints, switch the active generation between reads, and run the native case for both ordinary and split pages. Rule and prove either continuity with no missing or duplicate subject or an explicit reset that makes the caller restart the page sequence; a cursor must not silently change its generation meaning. + - Pagination across a generation switch. Keep two subscribed pages pinned with both cursor endpoints, switch the active generation between reads, and run the native case for both ordinary and split pages. Rule and prove either continuity with no missing or duplicate subject or an explicit reset that makes the caller restart the page sequence; a cursor must not silently change its generation meaning. 3b ruled continuity (`read-models.sdp.md`, `pageInGeneration`) and showed it at `convex-test` through both compositions' lists, the tie and a pinned page included, and on the native backend in the L2-5 run for an ordinary page; a split page across a switch has not run natively. - Sharpen Leads 1's overlapping paused rebuilds. Two paused rebuilds share one source scope, and the first resumes or aborts while the second still needs the pause. The native case must show either refusal of the second owner before it changes anything or independent ownership whose release cannot reopen the other rebuild's writers; a bare scope-wide Boolean is not proof of ownership. - Scheduled argument size. The scheduling documentation says 8 MB for aggregate arguments while the limits page says 16 MiB. Probe 7 resolved it on the pinned release: 16,777,216 bytes of Convex value bytes over one mutation's scheduled arguments, recorded on F13. @@ -435,8 +466,14 @@ Leads 30 to 38 are lane data's of the tooling unit, as `leads-data.md` in its fo - That a development deploy key with the fourteen actions covers every CLI call the harness makes, export and import included, is from source only; the first hosted run settles it. 40. From the check of the 3b memo, beside what rulings 2 took. - `fixture/convex/migrations.ts:51` spreads the projection's result after a `!`. TypeScript erases the `!`, so a `null` fails at the write and not there. Line 39 is a `typeof` type query, so the rename there changes a type expression too. - - Test files call `activate` directly, not only through the eight helpers: `tests/native/production-orders.test.ts:50`, `tests/native/read-models.test.ts:82`, `tests/simulator/read-model.test.ts:220` and on. 3b's setup change edits them. - `restore.sdp.md:82` does not say that history arrives by import, which the memo's question 8 read into it. +41. From round 3's unit 3b. Each is a cost note, a number still to take, a test gap or a comment, and none is on a path a healthy run or an ordinary failure takes. + - A purge needs one extra, empty batch for each tenant whose row count is an exact multiple of the cap (the proof lane's attack). + - The comparison run the ledger asked for beside the L2-5 run's reruns, which Q5 of the rulings declined; the reruns and their `occInfo` are counted in the run instead. + - A grant written before the tenant ID bound was enforced can hold a tenant ID above it, and `fillTenantsBatch` copies it as it is. The Spec makes the repair the operator's before the fill (G2); no such grant exists here, and the upgrade path is open. + - The `place-order-timing` pins moved with the slimmer row's bytes and have not run on a quiet machine; the file is skipped in the tier. + - What sol's last read left at `de285bb`: four mutations of the cursor codec survive (the `"undefined"` decoding, `jsonToConvex`, `convexToJson`, `sameValue`'s structural comparison), each unobservable while every composition's prefix is primitives; the codec's citation of `convex-helpers` ends at `stream.js:1447`, not `:1441`; the comment at `src/read-model/queries.ts:29` says "before a read" where the refusal follows the authorization and generation reads, and the `Value` type there does not admit the `undefined` sentinel; `read-models.sdp.md:78` says "either" over three fields. + - Four mutations survive the review: M06, an abort that no longer reopens gates, unreachable while the paused path is unbuilt; M25, the fixture's `rebuild` module removed while `snapshot-replacement` stays green, because that test installs no read model; M31, the left-behind progress count forced to zero at `tests/native/place-order-measurement.ts:257`, since `rowsLeftBehind` is never checked against it; M32, the three new tables dropped from Probe 7's fresh-import comparison at `tests/native/scheduler-composition.ts:121`, which stores no rows in them. The reports behind leads 1 to 3 and 6 are outside the repository, because they quote a private repository. The project notes file named in `AGENTS.md` lists them, with the reports, briefs and rulings of S0. From d2a8d1b8873f864181efec9c14bd8074ca42a7f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:13:38 +0200 Subject: [PATCH 111/155] design: sitting two, sixteen owner rows ruled and applied, the licence, the hosted key rotation recorded The owner's words: "Please help me resolve all these forks, terminology to unlock the mini roadmap and finalization of the current feature branch." and "I suggest you make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra with `gpt@libar-codex`. And rule." Ruled, from one recommendations page after one Opus and one astra read: OD-006 and OD-008 (the pass includes Sc ALL-1, each row at its contract tier), OD-138 (the mapping confirmed), OD-041 (a history-reading migration is written by the sweep alone, a command or read on an unreached stream refused as transient, D3 whole), OD-042 (fail closed, no override, the drill before every release), OD-114 (D13 stands), OD-044 (D2's premise corrected, F18's wording), OD-063 (a component per context), OD-048 (Apache-2.0), and the language rows OD-009, OD-022, OD-029, OD-139, OD-140 with OD-017 and OD-110 (CONTEXT.md leads; the refusal field is entry; tiers compiled, pure test, convex-test, native backend; backup archive; stated operator). The doc's two hunks wait for the owner's hand; the code follows in lane 4f. Co-Authored-By: Claude Fable 5.1 --- CONTEXT.md | 102 ++++++++- LICENSE | 202 ++++++++++++++++++ design/PLAN.md | 6 +- design/README.md | 10 +- design/ROADMAP.md | 13 +- design/STATE.md | 78 ++++--- design/decisions/register.json | 128 +++++------ design/reviews/consensus-ledger.json | 33 +-- design/specs/advanced/ordered-consumer.sdp.md | 3 +- .../specs/application/first-experiment.sdp.md | 10 +- design/specs/application/read-models.sdp.md | 2 +- design/specs/application/restore.sdp.md | 12 +- design/specs/application/write-pause.sdp.md | 2 +- design/specs/command/outcome-boundary.sdp.md | 12 +- design/specs/context/context-component.sdp.md | 8 +- design/specs/context/journal.sdp.md | 25 +-- .../specs/context/persistence-adapter.sdp.md | 4 +- design/specs/context/queries.sdp.md | 10 +- .../d02-context-owns-state-and-journal.sdp.md | 14 +- .../d13-deferred-work-is-an-obligation.sdp.md | 2 +- .../specs/facts/f18-commit-timestamps.sdp.md | 6 +- .../specs/obligations/do-nothing-check.sdp.md | 4 +- .../obligations/obligation-module.sdp.md | 4 +- .../transactional-domain-platform.sdp.md | 4 +- design/specs/platform/vocabulary.sdp.md | 33 ++- package.json | 1 + tests/pure/acceptance.test.ts | 3 +- 27 files changed, 524 insertions(+), 207 deletions(-) create mode 100644 LICENSE diff --git a/CONTEXT.md b/CONTEXT.md index 2075bda..54292c7 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -1,6 +1,6 @@ # Transactional domain platform -The language of a platform on which a business operation is one transaction: a command is authorized, decided against current state, recorded as events and answered, all or nothing. These are the words for what the platform is. Words for how it is designed and proven, and words for the work of building it, are listed apart at the end, because they must not leak into the first group. +The language of a platform on which a business operation is one transaction: a command is authorized, decided against current state, recorded as events and answered, all or nothing. These are the words for what the platform is. Words for how it is designed and proven, and words for the work of building it, are listed apart at the end, because they must not leak into the first group. This file leads: `spec:platform.vocabulary` follows it, and the doc keeps its authority over what the platform decides. ## Language @@ -22,6 +22,10 @@ _Avoid_: operation (alone), endpoint, handler, context command The parent's carrying out of one command across one or more contexts, with one outcome. _Avoid_: workflow, saga, orchestration, executor (the code's name for the function) +**Entry**: +A function the parent registers for a caller: a public command, an internal command, a parent query or an operator entry. +_Avoid_: endpoint, handler, function (alone) + ### Commands and outcomes **Command**: @@ -104,7 +108,7 @@ _Avoid_: apply, reduce, patch, state update **Rebuild**: Recomputing stored state from the history that should produce it: a stream's current state from its events, or a read model from its sources. -_Avoid_: replay, resync, reprocess, backfill (one step of a read-model rebuild) +_Avoid_: replay, resync, reprocess **Baseline event**: An event that holds migrated state, from which a rebuild starts when the meaning of earlier events has changed. Earlier events stay readable. @@ -124,6 +128,10 @@ _Avoid_: scope (alone), tenancy The server-established identity that caused a command: a human, a service, an agent, a reviewer or an operator. _Avoid_: user, caller, principal, identity +**Stated operator**: +The name an administrator states when calling an operator entry. It records who said they acted and establishes no actor; an actor of kind operator is one the server establishes, which no entry does. +_Avoid_: operator (alone, for this meaning), actor (for a stated name) + **Grant**: An authoritative statement that an actor holds a permission in a tenant, read in the same transaction as the command it authorizes. _Avoid_: role, ACL, access rule @@ -156,6 +164,18 @@ _Avoid_: replay, repeat, retry (what the caller does) ### Reads +**Parent query**: +A read the parent answers to a caller, authorized in the parent, relaying a context query or reading a read model. +_Avoid_: query (alone), endpoint, read (alone) + +**Context query**: +A read a context answers to the parent, returning a DTO and never a document. +_Avoid_: query (alone), getter + +**Query refusal**: +The refusal of a parent query, in the same wire shape as a rejection, naming the entry that refused. It is not an outcome, and nothing is recorded. +_Avoid_: rejection (for a query), error, denial + **Read model**: A stored shape made for reading, updated inside the command from the state that command committed. _Avoid_: view (alone), cache, table, projection (the logic, not the data) @@ -168,10 +188,64 @@ _Avoid_: read model (the data), handler, subscriber One numbered build of a read model. _Avoid_: version, attempt +**Installation**: +Making a read model exist in a deployment: its declaration and its first rebuild. +_Avoid_: activation, first activation, setup + +**Backfill**: +The part of a rebuild that writes a generation's rows from its sources. +_Avoid_: rebuild (for this part alone), replay, import + +**Verification**: +The part of a rebuild that checks a generation's rows against its sources and corrects them. +_Avoid_: validation, reconciliation, audit + +**Pass**: +One traversal of a rebuild over its sources or its rows: a backfill, a verification, a purge or a fill. The verb passes and a pass condition belong to scenarios, and a test's outcome is a test result. +_Avoid_: run, iteration, round + +**Switch**: +Making a verified generation the one readers see, in one write. +_Avoid_: cutover, promotion, activation + +**Rollback**: +Making a retired generation the one readers see again. +_Avoid_: revert, undo, downgrade + +**Retired generation**: +A generation readers no longer see, kept until it is purged. +_Avoid_: old generation, previous version, inactive + +**Purge**: +Deleting a retired generation's rows. +_Avoid_: cleanup, garbage collection, drop + +**Fence**: +The number a generation's batches carry so that a batch from before an interruption writes nothing. +_Avoid_: epoch, token, lease + +**Progress row**: +The row that holds where a rebuild stands, read by its batches and by no command. +_Avoid_: checkpoint, cursor row, status row + +**Tenant list**: +The parent's list of its tenants, which a rebuild and a sweep walk one tenant at a time. +_Avoid_: tenant table, registry, directory + **Write pause**: A stretch during which commands that write the sources of a read model are refused, so that the read model can be rebuilt from a consistent cut. _Avoid_: lock, maintenance mode, freeze +### Backups + +**Backup archive**: +The exported file of a deployment's table data at one moment, which a restore imports. +_Avoid_: snapshot (a transaction's view of the database), export, dump, backup (alone, for the file) + +**Restore**: +The procedure that imports a backup archive under matching code and proves the invariants hold before any writer runs. +_Avoid_: import (one step of it), recovery, rollback + ### Deferred work **Obligation**: @@ -206,9 +280,27 @@ Words for how the platform is specified and proven. They are lasting and may app - **Do-nothing option**: the design that adds no mechanism and relies on Convex as it is. - **Fact**: something the design relies on Convex to do, marked documented, probed or assumed. - **Probe**: one small test on a real backend that turns an assumed fact into a probed one. -- **Scenario**: a situation with a pass condition that the platform must meet. -- **Tier**: the kind of test that can prove a scenario: domain, simulator, native or end to end. +- **Scenario**: a row of the doc's acceptance table, a situation with a pass condition that the platform must meet; a case is one example of a row. +- **Tier**: the kind of test that can prove a scenario: compiled, pure test, convex-test or native backend. +- **Target**: where a native backend test runs, the local backend or a hosted deployment, recorded apart from the tier and the composition. - **Composition**: an application assembled from the platform for a purpose: the production composition, or the fixture composition that exists only to prove scenarios. +- **Verifier**: a test bound to an example, which proves it when it passes at the example's tier on its composition. +- **Acceptance check**: the check that reads from the graph every scenario the first experiment requires and from one run's record whether each passed. +- **Test result**: what one run of a verifier leaves in the run record: passed, failed or skipped. +- **Run record**: the harness's record of one run of the test projects, with the commit, the pins, the target and every test result; a restore's own record is a restore run. +- **Measurement record**: a run record's entry that holds a measured cost or time with its pins and its target. +- **Local backend**: the pinned backend process the harness starts and disposes of for a test. +- **Hosted deployment**: a Convex Cloud deployment the harness targets with a deploy key. +- **Deploy key**: a hosted deployment's credential, scoped to the actions a run needs. +- **Admin key**: the local backend's credential. +- **Usage reading**: what a hosted deployment's usage report says for a window; a difference of two readings is one run's own only when no other run overlapped it. +- **Stated plan**: the Convex plan a hosted run records as the owner stated it, never read from billing. +- **Kept dataset**: the data a hosted deployment carries from one run to the next on purpose. +- **Planting record**: the row a run writes beside a schedule it planted, so that a later run can tell a retained schedule from a lost one. +- **Execution**: one run of a function by the backend, the unit the function log counts. +- **OCC rerun**: an execution the engine repeats after a write conflict; neither an attempt nor a retry by the caller. +- **Contention**: several commands at once on one stream. +- **Controlled change**: one difference against a reference run, so that a cost is attributed to a line. ## Words that are not part of the language @@ -223,7 +315,7 @@ Words about the work of building the platform: when something was done, in what Three words exist on both sides and mean different things: - **Agent**: in the language, a kind of actor whose proposals the parent checks. In the work, a model that writes or reviews code. Only the first is a domain term. -- **Reviewer** and **operator**: in the language, kinds of actor. In the work, whoever reviews a change or runs a deployment by hand. +- **Reviewer** and **operator**: in the language, kinds of actor, and apart from them the stated operator. In the work, whoever reviews a change or runs a deployment by hand. - **Operation**: in the language, one accepted execution of a command. In the work, nothing: "an operation of the session" is a step. Fixture names such as depot, document and stock are the fixture composition's own small domain. They are not platform language either. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/design/PLAN.md b/design/PLAN.md index cc74c3c..8b69e96 100644 --- a/design/PLAN.md +++ b/design/PLAN.md @@ -682,11 +682,11 @@ Ranges for new extensions: B uses E-20 to E-29, C uses E-30 to E-39, D uses E-40 Recorded here and in README section 8. The corpus proceeds on the provisional reading shown; nothing is silently decided. -1. Vocabulary clashes (OQ6). The doc says to pick one meaning of "receipt" and "generation" before code names them. Provisional: receipt is the stored idempotency outcome; the row that proves an effect happened is "completion evidence"; generation is a numbered read-model build; an attempt number is "attempt". -2. Facts as constraints (OQ5). The plan maps ledger rows to `constraint` Specs with an evidence-status target. Confirm, or choose `rule` Specs instead; the change is mechanical. +1. Vocabulary clashes (OQ6). The doc says to pick one meaning of "receipt" and "generation" before code names them. Provisional: receipt is the stored idempotency outcome; the row that proves an effect happened is "completion evidence"; generation is a numbered read-model build; an attempt number is "attempt". Ruled: as provisional. +2. Facts as constraints (OQ5). The plan maps ledger rows to `constraint` Specs with an evidence-status target. Confirm, or choose `rule` Specs instead; the change is mechanical. Ruled: confirmed. 3. Plan-imposed numbering. F1 to F17, Sc L0-1 to ALL-1, OQ1 to OQ6 and E-numbers exist only in this plan. The doc should adopt them or the citations stay plan-relative. 4. Refused-command records (OQ2). Whether security audit or agent proposals need a record of refusals decides whether the generic internal dispatcher of D7 exists. The design carries it as a conditional. -5. Trivial contexts as plain tables (OQ1). The design assumes a component per context everywhere; the experiment decides. +5. Trivial contexts as plain tables (OQ1). The design assumes a component per context everywhere; the experiment decides. Ruled: a component per context, reopened only for a measured workload that misses an agreed target or a hosted usage reading above an agreed share. 6. One document per stream and the maximum order size (OQ3). The design assumes one document per stream and leaves the maximum lines as a product number with a placeholder. 7. Where the experiment lives (OQ4). Not decided by the corpus. 8. Readiness. No agent states `ready`. After consensus the owner may state it on Layer 0 to 2 Specs whose floor clears; recipe 9 lists the floor per Spec. diff --git a/design/README.md b/design/README.md index e4c4127..7659de1 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -232 specs · 5 packs · 131 anchors → 368 nodes · 1223 edges (0 errors, 0 warnings) +232 specs · 5 packs · 131 anchors → 368 nodes · 1225 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -220,7 +220,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-39 | [`command.idempotency-and-receipts`](specs/command/idempotency-and-receipts.sdp.md) | The second UI submit of a create is answered with the rejection `entityExists`, not an idempotent success, because the UI path writes no receipt to replay from and the client holds the entity ID | | E-40 | [`application.rebuild`](specs/application/rebuild.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | A batch is one mutation with one cursor on the progress row, at most 100 entities capped by the row budget and by the context's byte-derived page bound, a design number not attributed to the component; a self-scheduled internal mutation drives every batch, the backfill, the verify pass and the purge, and the rebuild uses `@convex-dev/migrations` for nothing; one batch names one tenant, taken from the tenant list; a fence that every interrupt, resume, rollback, abort and purge start raises | | E-41 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | The caller-driven import command with an `imports` progress row; the input bound itself is the declaration's `bounds.maxItems` under E-38 | -| E-42 | [`application.restore`](specs/application/restore.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | Four checks (journal, domain, read model, receipt) as checkpointed batches, the stream check bounded through the `rebuild` query's `maxEvents` so its worst case is one fold, and a refused fold counted as a journal finding; a fifth step that inventories the closed gate entries, the generations in flight and the running imports the snapshot restored; the `restoreRuns` record; an accepted branch that resumes every stranded generation, proven by the generation row's fence; `MAINTENANCE_MODE=restore` as the writer door, read by the write pause's check before the gate document and cleared before the durable profile's schedule rebuild runs | +| E-42 | [`application.restore`](specs/application/restore.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | Four checks (journal, domain, read model, receipt) as checkpointed batches, the stream check bounded through the `rebuild` query's `maxEvents` so its worst case is one fold, and a refused fold counted as a journal finding; a fifth step that inventories the closed gate entries, the generations in flight and the running imports the snapshot restored; the `restoreRuns` record; an accepted branch that resumes every stranded generation, proven by the generation row's fence; `MAINTENANCE_MODE=restore` as the writer door, read by the write pause's check before the gate document and cleared before the durable profile's schedule rebuild runs; a failed check keeps the whole deployment closed with no override; the drill before every release, its cadence reconsidered on the first drill's measured check times | | E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed, with `AuditRecordInput` and the source of each field in the command; the `operatorAudit` table for the gate changes an operator entry makes; Convex's own audit logging recorded as the do-nothing option; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | | E-44 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | `BusinessFailurePolicy` (`carry` or `reject`) per context call, applied in the executor body through the pinned helper `carryOrReject` | | E-45 | [`application.read-models`](specs/application/read-models.sdp.md) | At most 8 component reads per parent query and 4 read-model rows written per command, counted as the rows step 9 inserts, replaces or deletes and checked after each `streams` entry is applied, above which step 9 throws a plain error | @@ -255,11 +255,11 @@ The doc's own open questions: The plan's section 11, the ambiguities neither the doc nor SDP rules on, carried on the provisional reading shown: -1. Vocabulary clashes (OQ6), as above. -2. Facts as constraints (OQ5). Confirm, or choose `rule` Specs; the change is mechanical. +1. Vocabulary clashes (OQ6), as above. Ruled: receipt is the stored command outcome, generation a read-model build. +2. Facts as constraints (OQ5). Ruled: confirmed as mapped. 3. Plan-imposed numbering. F1 to F17, Sc L0-1 to ALL-1, OQ1 to OQ6 and the E-numbers exist only in the plan and this corpus; the doc should adopt them or the citations stay plan-relative. 4. Refused-command records (OQ2), as above. -5. Trivial contexts as plain tables (OQ1), as above. +5. Trivial contexts as plain tables (OQ1), as above. Ruled: a component per context. 6. One document per stream and the maximum order size (OQ3), as above. 7. Where the experiment lives (OQ4), as above. 8. Readiness. No agent states `ready`. After review the owner may state it on Layer 0 to 2 Specs whose floor clears; the Design Review index shows the floor each Spec reaches, and most Layer 0 to 2 Specs already reach `ready` structurally. diff --git a/design/ROADMAP.md b/design/ROADMAP.md index 597cce4..5952789 100644 --- a/design/ROADMAP.md +++ b/design/ROADMAP.md @@ -1,6 +1,6 @@ # Roadmap: the next units -Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of round 3's unit 3b on 2026-10-03. +Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of round 3's sitting two on 2026-10-03. ## How to read it @@ -25,14 +25,15 @@ The slices of `STATE.md` keep their names. S3 is round 1's attribution and round | 2 | g | Also done in round 2: the review's fold-in, the caller's signal in the harness, three `CancelOrder` tests, `fillTenants` and the resumed batch size on paper | mixed | operator, domain | under an evening | the review | | 3 | a | The gate, the restore door, and a gate change as audit that fails closed. Done in round 2: 106 proof tests written apart from the fix, all passing at the merge, the fix wrong nowhere | mixed | operator | an evening | 2b | | 3 | b | The online rebuild of the order summary, Sc L2-5 at the native tier: thirteen functions per composition, laid out in `layout-3b.md` of the tooling unit as three lanes and 33 proof obligations | build | domain | a night | 2b, 2d, 3a, the layout's eight questions ruled, the five register rows recorded | -| 3 | e | The hosted tier, commissioned from `design-hosted-tier.md` of the tooling unit: four hosted examples on one shared development deployment, the local backend primary | build | convex, operator | unknown until the first run | 3b, the deploy key and its secret, `patch-hosted-specs.diff` applied, sitting two's words | +| 3 | e | The hosted tier, commissioned from `design-hosted-tier.md` of the tooling unit: four hosted examples on one shared development deployment, the local backend primary | build | convex, operator | unknown until the first run | 3b, the deploy key and its secret (done), sitting two's words (done), `patch-hosted-specs.diff` applied with its two word changes | | 3 | c | The two harness reads and the fact rows no Spec carries. Done in round 2: F18 to F20 probed, the "no job" assertions read `_scheduled_functions`; the deployed-function read stays deferred | mixed | convex | a session plus an evening | nothing | -| 3 | d | Sitting two: the language, the pass sentence, and the S5 rows | decide | product, domain, operator | an evening of forks, one check, one owner sitting | nothing | +| 3 | d | Sitting two: the language, the pass sentence, and the S5 rows. Done: sixteen rows ruled from one recommendations page after one Opus and one astra read | decide | product, domain, operator | an evening | nothing | | 4 | a | A command's audit fails closed and its diagnostic never aborts, Sc ALL-1 | mixed | operator | an evening | 3a | | 4 | c | The history view's fold, its memory and its rollback, on paper | design | domain | an evening | 2c | | 4 | d | The receipts sweep, one tenant per run | build | operator | under an evening | the tenant list from 2b | | 4 | e | `ready` on the built Specs, by family | decide | product | a session, one owner unit | 1c, sitting two | -| 5 | a | Baselines, migration and the fold bound, Sc L2-7 at the native tier | mixed | domain | a night or more | OD-041 from sitting two, 3b | +| 4 | f | The language in the code: the refusal field `entry`, the tier names in the examples' opening lines and the harness parser, the harness's backup archive names | build | product | under an evening | sitting two | +| 5 | a | Baselines, migration and the fold bound, Sc L2-7 at the native tier; the history-reading sweep with its transient refusal and the restore check's pending-work inventory, at `convex-test` for the interleavings and natively for the sweep | mixed | domain | a night or more | OD-041 (ruled), 3b | | 5 | b | The restore procedure and the drill, Sc L2-8 | mixed | operator | unknown until 4b | 4b, 5a | | 5 | c | The first adopter-facing cut: README, the example walkthrough, the package shape | mixed | product | a night | OD-047, OD-048, Sc L2-5 to L2-8 bound | | 5 | d | The build of the order allocation history view, Sc L2-6 at the native tier, by ruling F10 | build | domain | a night | 3b, 4c | @@ -70,8 +71,8 @@ Done on 2026-10-02 and 2026-10-03, as nine lanes merged into `feature/round-2`, The owner discusses the next session's work before it is prepared. In this order: 1. **3b, the online rebuild of the order summary**, is done, on 2026-10-03: thirteen functions per composition, every read model installed through its first rebuild, the parent list paged across a switch, and Sc L2-5 passing at the native tier on the production composition. `STATE.md`, "Where the work stands", has its numbers: 325 of 325 proof tests at the merge, the L2-5 run's times, the review and three reads of the fold-in. -2. **The hosted tier**, a build unit commissioned from `design-hosted-tier.md` of the tooling unit, is next. Of its four prerequisites, the environment `hosted` with its three variables and the deploy key `hosted-ci` with its secret exist since 2026-10-03; the key check, one command, and the dashboard check are the owner's; `patch-hosted-specs.diff` is applied first in the unit, after sitting two rules the words it writes. It shows the four hosted examples run by the workflow on `main`: F16's retention, F12's in-place import, the production composition's acceptance and Probe 3's function calls. The local backend stays primary. -3. **3d, sitting two.** The language rows OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added (snapshot, tenant fill, the measurement's words) and those of the tooling unit (R5's four, the hosted patch's, 3b's, the operator term); OD-008 with OD-006; OD-138; OD-041, OD-042 and OD-114, the last with Probe 7's facts; OD-044 with F18 against D2; OD-063 with its local numbers; and OD-048, the licence. The F2 comparison is done: every target held on the quiet ThinkPad. +2. **The hosted tier**, a build unit commissioned from `design-hosted-tier.md` of the tooling unit, is next. Its prerequisites are met but one: the environment `hosted` with its three variables and the deploy key `hosted-ci-scoped`, scoped to the fourteen actions and checked, exist since 2026-10-03; sitting two ruled the words; `patch-hosted-specs.diff` is applied first in the unit, with its "marker" read as planting record and its test "pass" as test result. The one thing still the owner's is the dashboard check: `clean-capybara-16` is a production deployment the design does not want. It shows the four hosted examples run by the workflow on `main`: F16's retention, F12's in-place import, the production composition's acceptance and Probe 3's function calls. The local backend stays primary. +3. **3d, sitting two**, is done, on 2026-10-03: sixteen rows ruled (the five language rows with OD-017 and OD-110, OD-006 with OD-008, OD-138, OD-041, OD-042, OD-114, OD-044, OD-063 and OD-048), each applied at the Spec that owns it, `CONTEXT.md` carrying the words. The method: one recommendations page by the main thread, one Opus and one astra read of it, then the ruling. Two doc hunks wait for the owner's hand, and the language in the code is lane 4f. 4. **4a, command audit and the diagnostic.** 3a has built `writeAudit` and step 10, so what is left is the Spec text on which outcomes emit a record and where a duration comes from, the sink, and Sc ALL-1 bound. 5. **4c, the history view on paper.** 2c is done, so `OrderCancelled` and `AllocationReleased` exist for its fold. 6. **4d, the receipts sweep.** The tenant list and `nextTenant` are built. diff --git a/design/STATE.md b/design/STATE.md index f62163a..fa674a2 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -1,25 +1,25 @@ # State of the design corpus -Written on 2026-10-03 at the close of unit 3b of round 3 of the roadmap, which followed the tooling unit, round 2, round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. +Written on 2026-10-03 at the close of sitting two, unit 3d of round 3 of the roadmap, which followed unit 3b, the tooling unit, round 2, round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. ## Measured at close ``` -232 specs · 5 packs · 131 anchors → 368 nodes · 1223 edges (0 errors, 0 warnings) +232 specs · 5 packs · 131 anchors → 368 nodes · 1225 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings; readiness divergence: [] open questions: 69 Specs, 147 questions, 9 blocking; extensions registered: 53 stated readiness: {'defined': 222, 'idea': 1, 'scoped': 9} -ledger: 278 findings, {'fixed': 148, 'open': 99, 'owner': 1, 'partially-fixed': 30} -open findings by slice: {'L3': 31, 'P': 30, 'S3': 1, 'S4': 15, 'S5': 22} -corpus digest: 8e9818820eb59f6b +ledger: 278 findings, {'fixed': 152, 'open': 94, 'owner': 1, 'partially-fixed': 31} +open findings by slice: {'L3': 30, 'P': 29, 'S3': 1, 'S4': 15, 'S5': 19} +corpus digest: 94bb9c63bfd80580 protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` -`python3 design/tools/decisions.py`, after the register patch of this unit (`314c265`); the close applies none of its own: +`python3 design/tools/decisions.py`, after sitting two's patch of sixteen rows: ``` total: 154 -status: unsorted=0, sorted=96, waiting=22, decided=21, folded=15 +status: unsorted=0, sorted=82, waiting=20, decided=37, folded=15 class: tactical=3, delegated=86, owner=50, null=15 advisor: convex=36, domain=45, operator=34, product=39 ``` @@ -34,7 +34,15 @@ From this merge on, a unit reached `main` through a pull request, tactical decis ## Where the work stands -Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, round 1 of that roadmap is on `main`, round 2 is on `feature/round-2`, a design unit for the tooling followed on the same branch, and so did round 3's first unit, 3b, the online rebuild of the order summary. +Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, round 1 of that roadmap is on `main`, round 2 is on `feature/round-2`, a design unit for the tooling followed on the same branch, and so did round 3's first unit, 3b, the online rebuild of the order summary, and then sitting two, the owner's decisions of round 3. + +Sitting two, round 3's unit 3d, on 2026-10-03 from about 08:00 to 09:30, on the ThinkPad, on `feature/round-2` from `840b32c`. The owner brought a table of recommended resolutions for every row and said "Please help me resolve all these forks, terminology to unlock the mini roadmap and finalization of the current feature branch." The main thread first prepared a rulings page and five lanes, and the owner stopped it: "this is excessive", then "I suggest you make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra with `gpt@libar-codex`. And rule." Decision 50 holds the method. What it did, at the tier named: + +- The recommendations, one page in the unit folder, `recommendations-sitting-two.md`, nine subjects over sixteen rows, with the one Convex claim checked by the main thread: docs.convex.dev's commit-timestamp page, read 2026-10-03, documents `ctx.meta.getSnapshotTs()` as the reader's upper bound, and the public declarations of the installed `convex` 1.46.0 omit it (compiled tier, a grep of the declarations). +- Two reads of the page, the same brief, read only: an Opus agent in 11 minutes and `gpt-6-astra` at xhigh in 5 minutes. Both agreed with every recommendation and asked for changes on four: the pure test that pins twenty-three rows (`tests/pure/acceptance.test.ts`) had to change with `acceptanceRows`; the migration ruling as recommended, a write pause of the source scope for a history-reading sweep, was new mechanism (an operator cannot close a source scope: `write-pause.sdp.md`, `src/gate/entries.ts`, held at pure test and `convex-test`), wider than the per-stream refusal E-25 rejected, and outside D9's reason for a pause, and it left reads during the sweep and a backup taken mid-migration unanswered; the 3.17 ms figure is F19's probe on the fixture and the hosted quota is still unread; `functionName` is Convex's own word for a mechanism and the field carries a declaration's or a policy's name; "scenario", "run record" and the noun "pass" needed their carve-outs. Astra's correction that the on-load migration describes the Spec and not the code holds: `src/context/adapter.ts` rejects a schema-version mismatch and migrates nothing. +- The rulings. The main thread took every recommendation with the reviewers' changes, and changed the migration ruling to the narrower form: a history-reading migration is written by the sweep alone, a command or a read on a stream the sweep has not reached is refused as transient (`writePaused`), no stream type is paused, the restore's journal check inventories `awaitingBaseline` as pending work and the accepted branch restarts the sweep. The refusal field is `entry`, the name of the entry (public command, internal command, parent query or operator entry) that refused. Sixteen rows are `decided` by the owner with the rulings in `register.json`, applied by the tool from `patch-register-sitting-two.json` in the unit folder; no new fork page was written, since the owner ruled from the recommendations. +- Applied at the Spec that owns each question, by the main thread: `first-experiment` (the pass rule, `acceptanceRows` with Sc ALL-1, E-17's question), `transactional-domain-platform` (OQ5), `journal`, `persistence-adapter`, `queries`, `restore` and `write-pause` (OD-041 and OD-042, the gate's migration clause cut), `d13`, `do-nothing-check` and `obligation-module` (Probe 7's questions), `d02`, `journal`, `f18` and `ordered-consumer` (D2's premise, with `constrainedBy` edges to F18), `context-component`, `d02`, `first-experiment` and `obligation-module` (OQ1), `vocabulary`, `outcome-boundary`, `queries` and `read-models` (the language), `CONTEXT.md` (twenty-nine entries added or changed), `PLAN.md` 11 and `README.md` section 8 and the E-42 row, six ledger rows, `LICENSE` and the `license` field. `check.py` OK; `npm test` 63 files, 956 tests, at the pure, `convex-test` and compiled tiers. +- Not changed: the doc, whose two hunks are the owner's (`patch-doc-sitting-two.diff` in the unit folder); the code, where the refusal field, the tier names in sixty example opening lines and the harness parser, and the harness's backup archive names now disagree with the Specs by ruling, as roadmap lane 4f; the hosted patch, applied in the hosted unit with "marker" read as planting record and a test's "pass" as test result. Round 3, unit 3b, `application-platform-round-3`, on 2026-10-03 from about 03:32 to 07:00, on the ThinkPad, from `feature/round-2` at `62b8817` to `399b410`. The owner's words at the opening: "please plan and execute the scope, including validaiton of the hosted convex env. use fable 5.1 for designs of bigger pieces of solution, opus, astra and sol for execution." The hosted check at the opening was read only: the environment `hosted` shows its three variables and its secret, and `convex env list` with the owner's login reached `wandering-mule-403`; whether the key itself authenticates is the owner's one command (Owner queue). During the unit, at the integration, the owner asked whether the proof records were needed: "Tests should be all the proofs we need." and "We need to weed out all examples from the codebase and do all the suggested things." Decision 48 holds it. The main thread wrote two rulings pages, `rulings-1-round-3.md` and `rulings-2-on-the-integration.md`, and `answers-lanes.md` keeps every answer with the ruling after it. What each lane showed, at the tier named: @@ -220,17 +228,15 @@ Thirty more findings are `partially-fixed`, each waiting for its slice. Remove a ## Next unit -`ROADMAP.md` is the order, ruled as cut by decision 36. Rounds 1 and 2 are done, and so is 3b, the first item of round 3. The next unit is the hosted tier, round 3's item 2, a build unit commissioned from `design-hosted-tier.md` in the tooling unit's folder: its build cut, its order and its "built when" list. The owner's words of 2026-10-03: "Local backend should be primary, IMO. But happy to take the hosted as default if easier for start." So the local backend stays the primary target, and the hosted deployment is a target of the native tier on a schedule, never in `test:all`. Before it starts: +`ROADMAP.md` is the order, ruled as cut by decision 36. Rounds 1 and 2 are done, and so are 3b and sitting two, round 3's first and third items. The next unit is the hosted tier, round 3's item 2, a build unit commissioned from `design-hosted-tier.md` in the tooling unit's folder: its build cut, its order and its "built when" list. The owner's words of 2026-10-03: "Local backend should be primary, IMO. But happy to take the hosted as default if easier for start." So the local backend stays the primary target, and the hosted deployment is a target of the native tier on a schedule, never in `test:all`. Before it starts: - Done: the setup, which made the project `libar-platform-hosted` and its development deployment `dev/hosted` (`wandering-mule-403`). -- Done by the owner at the owner's own prompt on 2026-10-03: the GitHub environment `hosted` with its `main` branch policy and the three variables (`HOSTED_DEPLOYMENT_URL` as the printed URL with the region, `HOSTED_DEPLOY_KEY_NAME` as `hosted-ci`, `HOSTED_PLAN` as `Professional`), the deploy key `hosted-ci` made in the dashboard on `wandering-mule-403` (its value begins `dev:wandering-mule-403|`, the owner's reading), and the secret `HOSTED_DEPLOY_KEY` in that environment. `gh secret list --env hosted` and `gh variable list --env hosted` showed them. -- Still the owner's: the key check, one command in a shell that holds the key (Owner queue), which shows that `hosted-ci` authenticates against `wandering-mule-403`. -- Still the owner's: the dashboard check. The project showed a second development deployment, `clean-capybara-16`, which no agent created and which is most likely the owner's personal dev deployment from the dashboard's first open; if nothing uses it, the owner deletes it so that the project holds `dev/hosted` only, no production deployment and one key. -- First in the unit: `patch-hosted-specs.diff` applied, after sitting two rules its words. +- Done by the owner at the owner's own prompt on 2026-10-03: the GitHub environment `hosted` with its `main` branch policy and the three variables (`HOSTED_DEPLOYMENT_URL` as the printed URL with the region, `HOSTED_PLAN` as `Professional`), and the secret `HOSTED_DEPLOY_KEY` in that environment. +- Done on 2026-10-03, at the owner's request, as the owner's inspection of the dashboard and GitHub recorded it (`tmp-convex-findings.txt`, the owner's file): the first key, `hosted-ci`, carried 25 allowed actions where the design calls for 14, so it was replaced by `hosted-ci-scoped` with exactly the 14, the secret was replaced through stdin (GitHub's update time 2026-10-03T06:31:36Z), `HOSTED_DEPLOY_KEY_NAME` is `hosted-ci-scoped`, and `hosted-ci` was revoked; one key remains. The key-only check passed with CLI 1.46.0: `convex env list --names-only` in a fresh temporary home with no stored login. The CLI refuses an empty directory, so the check needs a minimal `package.json` naming `convex` 1.46.0 beside it, with no install; the Owner queue's procedure below says so now. +- Still the owner's: the dashboard check. `clean-capybara-16` is a production deployment, reference `production`, not a development one as this file said before; never deployed, no tables, no keys, created by the project's default. The design wants `dev/hosted` only, so the owner deletes it if nothing uses it. Deletion is permanent and an agent does not do it. +- First in the unit: `patch-hosted-specs.diff` applied, with sitting two's two word changes: its "marker" is a planting record, and a test's "pass" is a test result. The regional URL stays the printed form, and the selector's check that key and URL name one deployment must accept it (lead 39). -Its first runs are the owner's from this machine. The four hosted examples run by the workflow once it is on `main`, because GitHub runs a schedule from the default branch. - -Then sitting two, roadmap 3d, as the Owner queue lists it. Its ruling on the hosted patch's words comes before that patch is applied, so the owner may take it first. +Its first runs are the owner's from this machine. The four hosted examples run by the workflow once it is on `main`, because GitHub runs a schedule from the default branch, and the `hosted` environment admits `main` only, so the first hosted CI run is a step after the reviewed branch reaches `main`. `slice` S3 is round 1's attribution and round 2's measurement, the quiet timings included. S4 is round 2's paper and gate, round 3's online rebuild, and the builds of round 4. S5 is round 5. The paper decisions S4's rebuild needs are taken; those of the history view and of S5 are listed below. @@ -246,7 +252,7 @@ What S2 left, as it stands now: Decisions each later slice needs on paper before its code: - S4, the history view (roadmap 4c): order-independent cross-stream folds and whether deletion keeps fold memory (`r3-architecture-history-fold-order-and-deletion`); the paused path's cursor across sources, its verify pass and its batch bound; the L2-6 interrupt example's "can roll back", which the rebuild Spec now answers with a refusal. The rest of S4's paper was taken in round 2: `design-s4-paper.md` sections 1 and 7. -- S5: history-reading migrations off the command path unless the owner changes D3 (`r3-fidelity-migrate-on-load-replays-history-against-d3`); whether paused migrations are supported (`r3-fidelity-baseline-sweep-and-the-pause-contradict`); reopening authority, failure scope and drill cadence as explicit extensions (`r3-fidelity-restore-policies-cited-to-the-doc`); which diagnostic outcomes are emitted and where a duration comes from; what a restore in place does with the schedules it keeps (`probe-7-architecture-restore-kept-schedules-not-reconciled`); how the restore check detects a stream state that is not the fold of its events (`s3-measurement-architecture-current-state-not-checked-against-event-fold`); the read-only comparison of the restore's read-model check (`s4-paper-sdp-restore-read-model-check-compare-mode-undefined`). +- S5: the history-reading sweep's transient refusal and the restore check's pending-work inventory, ruled at sitting two (OD-041) and designed with the baseline unit; which diagnostic outcomes are emitted and where a duration comes from; what a restore in place does with the schedules it keeps (`probe-7-architecture-restore-kept-schedules-not-reconciled`); how the restore check detects a stream state that is not the fold of its events (`s3-measurement-architecture-current-state-not-checked-against-event-fold`); the read-only comparison of the restore's read-model check (`s4-paper-sdp-restore-read-model-check-compare-mode-undefined`). What the platform review of 2026-10-02 asks of the later units. The ledger owns each finding's counterexample and what its build must show. This is the order and the guidance that fits no single finding. @@ -265,7 +271,7 @@ What the platform review of 2026-10-02 asks of the later units. The ledger owns ## Tactical decisions taken on 2026-10-01 -The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words, 46 and 47 during the tooling unit on 2026-10-03, and 48 and 49 during round 3's unit 3b the same day. Any of them can be reopened. +The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words, 46 and 47 during the tooling unit on 2026-10-03, 48 and 49 during round 3's unit 3b the same day, and 50 and 51 at sitting two the same day. Any of them can be reopened. 1. Slices replace the eight fix units of the previous plan. 2. A slice is built on Specs at `defined` with no open blocker inside the slice. The owner states `ready` after the slice passes. @@ -316,6 +322,8 @@ The owner accepted the first nine on the session's recommendation. The session t 47. No agent of this harness writes a secret. On 2026-10-03 the main thread's harness refused a secret-store write even through an agent, a "Secret-Store Writes" denial, and the main thread does not work around it. The owner creates a deploy key by hand and stores it at an interactive prompt, never in a chat or in a file in the repository. The setup around the key (the project, the deployment, the GitHub environment and its variables) is an agent's work. The owner's words: "This setup cannot be documented and executed by an agent? With gh cli, convex cli, and other tooling?" and "1. and if someting is an issue, i will jump in." When an agent's own permission check refuses a step, as it refused the setup lane's `gh api` write, the agent stops and hands the owner the exact commands. 48. Tests are the proof. The owner's words of 2026-10-03, during 3b: "Tests should be all the proofs we need." and "We need to weed out all examples from the codebase and do all the suggested things." What stays: one rulings page per unit, an answer of at most fifteen lines from each agent, one run-log line per GPT job in the project notes, this file, a design note when a Fable agent decides, the ledger and the register, and the harness's run records under `evidence/runs/`, which the acceptance check reads. What goes: `record-*.md` files, saved copies of reports and gate logs, a command per number, and a test shown to fail under a mutation outside the review lane; mutation testing is the reviewer's, in its own report. The repository's gates, which CI runs on every push, are the proof, so a fold-in runs the files it touches alone and CI's run on the push is the whole tier. It is written in `AGENTS.md`, `SESSIONS.md` and the orchestration guide, commit `e377f58` and its twin in the context repository. 49. The order of the work after the lanes. The integrator starts in two stages while a lane still runs: it merges the lanes that are done, rules their failures, and takes the last lane's head by message. The review starts on the merged head before the fold-in, so that its mutations and its native runs overlap the fold-in, and the same job is sent each fold-in head by message and reads the diff from the merged head to it. +50. An owner sitting is one recommendations page, two reads, then the ruling. The owner's words of 2026-10-03, after stopping a rulings page with five lanes ("this is excessive"): "I suggest you make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra with `gpt@libar-codex`. And rule." So the main thread writes one recommendation per row in the unit folder, one Opus agent and one `gpt-6-astra` job at xhigh read it on the same read-only brief, and the main thread rules and applies in its own thread. No fork page is written for a row the owner rules this way; the ruling lives in the register row, and the recommendations and both reads in the unit folder. The rows are recorded as the owner's, as decision 40 did. +51. A paper sitting renames nothing in the code. The words sitting two ruled leave the refusal field, the tier names in the examples' opening lines and the harness parser, and the harness's backup archive names disagreeing with the Specs on purpose, as roadmap lane 4f, so that the branch the owner reviews stays green and small. ## Owner queue @@ -328,31 +336,15 @@ class: tactical=3, delegated=86, owner=50, null=15 advisor: convex=36, domain=45, operator=34, product=39 ``` -The owner's one command before the hosted tier: the check that the deploy key `hosted-ci` authenticates against `wandering-mule-403`. It needs the key in the owner's shell, so it is the owner's, in an empty directory and read only. Paste the key at the silent prompt: - -``` -cd "$(mktemp -d)" && read -rs CONVEX_DEPLOY_KEY && export CONVEX_DEPLOY_KEY -HOME=$(mktemp -d) ~/dev-libar/libar-platform/node_modules/.bin/convex env list --names-only; unset CONVEX_DEPLOY_KEY -``` - -An empty list with no error means the key authenticates. The dashboard check is the owner's too ("Next unit"). - -Decided since the last close: OD-057 by the convex advisor, and OD-071, OD-075 and OD-080 by the domain advisor, each with its lean checked in the tooling unit as the ruling, which the owner agreed to ("I agree with all recommendations"). The fold-in of 3b wrote the patch, the agent's classifier refused `decisions.py apply`, and the main thread applied it through the tool with the decider corrected to each row's advisor, `314c265`. At the close before, OD-010 and OD-014 were decided by the domain advisor, and OD-057, OD-071, OD-075, OD-080 and OD-149 took the advisor's new leans, the first four with `checkedBy` from sol's recheck. OD-149 stays `sorted` and its `checkedBy` null: its `ifWrong` says a tenant missing from the list gets no row until `fillTenants` and a rebuild, while a live write in the active generation inserts a missing row (`projection-contract.sdp.md:49`). That changes no `do`, but the register's form for `checkedBy` is "every checked claim held". The hosted patch, when it is applied, adds OD-155, the hosted tier's money, as an `owner` row. +The key check is done ("Next unit"). For the record, the procedure that worked with CLI 1.46.0, which refuses a directory with no `package.json`: in a fresh temporary directory, write `{"name":"libar-hosted-key-check","private":true,"dependencies":{"convex":"1.46.0"}}` as `package.json`, install nothing, paste the key at a silent `read -rs CONVEX_DEPLOY_KEY`, export it, run the repository's `node_modules/.bin/convex env list --names-only` under a fresh temporary `HOME`, then unset the variable. An empty list with no error means the key authenticates. -At the close before, sitting one's twelve `owner` rows were recorded as the owner's (decision 40), and OD-007 was decided by the convex advisor. OD-154, the grant read, is a `delegated` row with its measured cost. +What waits for the owner after sitting two: -Sitting two waits for the owner. Its forks are drafted the same way as sitting one's when the owner asks: +- The doc's two hunks, from OD-008 and OD-044, in `patch-doc-sitting-two.diff` of the unit folder: `git apply ~/dev-libar-supporting-context/calibration/application-platform-sitting-two/patch-doc-sitting-two.diff` from the repository root, then a commit. Only the owner edits the doc. +- The dashboard deletion of `clean-capybara-16` ("Next unit"). +- The next language rows, words the ruling did not cover, which keep the code's names until the owner rules: interrupt, park, stall, batch chain, fill, the aggregate's marker, history view and history projection, tombstone, drill, door, gate, audit record, diagnostic, adopter, package, activation record. -- `OD-048`, the licence, the one fork of sitting one left the owner's. It blocks nothing before round 5c. -- `OD-044`, D2's premise, with F18 against it: on the pinned backend one transaction's rows share one commit timestamp in the parent and in components and later transactions get larger ones, while D2 and `spec:context.journal` say no order token for the whole deployment exists. Nothing uses the internal `getSnapshotTs()`. -- `OD-063`, OQ1, a component per context, with its local numbers on the lean: a fixture component call cost 3.56 and 5.76 ms over a helper call in two readings under load, and Probe 11 read 7.6 to 15.5 ms per empty component call, all at the native backend tier on the local backend; on the quiet ThinkPad, 3.17 ms per component call over a helper call. -- `OD-114`, D13's do-nothing check, with Probe 7's facts: a restore in place keeps schedules the restored data no longer explains, a fresh restore keeps none, and a scan of `_scheduled_functions` stops at about 4,088 outstanding rows on the read limit. -- The language rows, OD-009, OD-022, OD-029, OD-139 and OD-140, with the words round 2 added: "snapshot" for the exported archive, which `CONTEXT.md` lists to avoid for state; "tenant fill" for `fillTenants`, `fillTenantsBatch` and `tenantFill`; the measurement's words (measurement record, cell, execution against attempt, rerun, contention, controlled change, local backend against hosted deployment); and the words of `ROADMAP.md`'s list. The tooling unit adds four sets: - - R5's four, which the unit used as its rulings defined them: local backend, the pinned process this harness starts; hosted deployment, a Convex Cloud deployment; deploy key, a hosted deployment's credential; admin key, the local backend's. - - The hosted patch's new words, which wait for this sitting before the patch is applied: the fields `target`, `statedPlan`, `ranBy`, `hosted` and `windowCrossed`; the types `HostedTarget`, `HostedRunFacts` and `HostedUsage`; the Design key `hostedProbeCopy`; plan and plan class; stated plan; usage reading; kept dataset; planting record; hosted run; preview deployment. Two clashes come with them: "marker", which already names an aggregate's marker, and "pass", which the rebuild's progress row already uses. - - 3b's words, from `layout-3b.md`'s last section: backfill and the backfill mode (which `CONTEXT.md:107` lists to avoid), verify pass, switch, rollback, retired generation, purge, install or first rebuild, fence, batch chain, progress row, pass, the role of a written generation, interrupt, park, stall, tenant list. - - The operator term: `CONTEXT.md:226` makes the operator a kind of actor, while the S4 paper's ruling 4 says no actor of kind `operator` is established and an operator entry takes an `operator` string. Until the owner rules, `CONTEXT.md` is read as ruling 4 says. -- OD-008 with OD-006, OD-138, and the S5 rows OD-041 and OD-042, as round 3's sitting two cuts them. +Decided since the last close: sixteen `owner` rows at sitting two, recorded as the owner's with the rulings in the rows (decision 50): OD-006, OD-008, OD-009, OD-017, OD-022, OD-029, OD-041, OD-042, OD-044, OD-048, OD-063, OD-110, OD-114, OD-138, OD-139 and OD-140. `decisions.py --decided --by owner --since 2026-10-03` lists them. Before that, in 3b and the tooling unit, OD-057 by the convex advisor and OD-010, OD-014, OD-071, OD-075 and OD-080 by the domain advisor, each with its checked lean as the ruling; OD-149 stays `sorted` with its `checkedBy` null, because its `ifWrong` did not hold. At sitting one, twelve `owner` rows were recorded as the owner's (decision 40), and OD-007 was decided by the convex advisor. OD-154, the grant read, is a `delegated` row with its measured cost. The hosted patch, when it is applied, adds OD-155, the hosted tier's money, as an `owner` row, and OD-036, Probe 3's quota half, takes its lean from that patch. `delegated` rows that wait for the next sort of their lens, each with the round's result on its lean: OD-066 (3N + 5 with every term attributed; the journal tail read is 10 documents and 6,010 bytes, 29 percent of the documents read on ten lines), OD-120 (Probe 7's states) and OD-154 (3 documents and 918 bytes for the grant read) for the convex lens, and OD-081 (Probe 7's restore answers) for the operator lens. OD-057 has left this list: its new lean closes on Probe 6's result and is checked. The session may not decide a `delegated` row (`decisions.py`: "session may decide only tactical rows"). @@ -495,3 +487,9 @@ The doc is `docs/convex-transactional-domain-platform-decisions.md`, unchanged s - `.claude/agents/fable-xhigh.md` is the agent definition the round-3 reviewers ran as. - `.claude/agents/advisor-convex.md`, `advisor-domain.md`, `advisor-operator.md` and `advisor-product.md` are the definitions of the four advisors. Each holds three paths and nothing else. - `design/generated/` is ignored and regenerated by `npx sdp view design`. +42. From sitting two. Each is a gap between a ruled Spec and the code, taken by lane 4f, or a word still without a ruling. + - The refusal field: the Specs say `entry`, the code and its tests say `commandType` (`src/command/outcome-boundary.ts`, `src/command/pipeline.ts`, `src/command/authority.ts`, the compositions' queries and about ninety test files). `commandType` stays on receipts, declarations, the idempotency key, audit records, the envelope's cause and the dispatcher. + - The tier names: sixty example opening lines say `domain tier`, `build tier`, `native tier` or `end to end tier`, and `harness/acceptance.ts` with `tests/pure/acceptance.test.ts` parses those words; the ruled names are pure test, compiled, native backend, with composition and target apart. Eight non-example Specs and `design/README.md:133` use the old names too. The vitest project names (`pure`, `simulator`, `native`, `types`) are the harness's and stay. + - The harness's `snapshot` members (`harness/snapshot.ts`, `exportSnapshot`, `replaceSnapshot`, `deploySnapshotFixture`, the probe-7 and `snapshot-replacement` tests): the Specs say backup archive. The hosted unit rewrites those paths and takes the names with it; the restore Spec's own "snapshot" sentences are read as backup archive until that unit edits them. + - The on-load migration is unbuilt either way: `src/context/adapter.ts` rejects a schema-version mismatch with a plain error, which is the ruled answer for a history-reading step and not yet the state-only chain of step 3a. + - Words without a ruling, which keep the code's names: interrupt, park, stall, batch chain, fill, the aggregate's marker, history view and history projection, tombstone, drill, door, gate, audit record, diagnostic, adopter, package, activation record. diff --git a/design/decisions/register.json b/design/decisions/register.json index 07a3bdd..1e7a12b 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -216,7 +216,7 @@ "provisionalReading": "Taken: no, the 23 rows of Layers 0 to 2.", "blocks": null, "advisor": "convex", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1: settling it reads the doc's scenario table, whether Sc ALL-1's 'every installed layer' counts as a Layer 0 to 2 row [design/specs/application/first-experiment.sdp.md:65]; a reading of the doc is the owner's.", @@ -225,9 +225,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "The experiment's pass includes Sc ALL-1: acceptanceRows lists it after Sc L2-9, and its two examples count as missing until the command audit and the diagnostic bind them. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-007", @@ -296,7 +296,7 @@ "provisionalReading": null, "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, settling it edits the doc: the doc's sentence 'passes when every Layer 0, 1 and 2 scenario passes on a native backend' (docs/convex-transactional-domain-platform-decisions.md:479) against the contract's tiers for Sc L0-1, L0-2 and L2-4 (design/specs/platform/acceptance-contract.sdp.md:58,62).", @@ -305,9 +305,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "The experiment passes when every Layer 0, 1 and 2 scenario and Sc ALL-1 passes, each at the tier the acceptance contract names for it; the doc's pass sentence changes by the owner's hand (patch-doc-sitting-two.diff). Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-009", @@ -327,7 +327,7 @@ "provisionalReading": null, "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, it rules on terms of CONTEXT.md: acceptance check, verifier and build tier are missing, scenario and run record each carry two meanings, and the tiers have three sets of names (design/STATE.md:220); the one code name in the bullet, otherPrincipalId, is a name inside the language and goes with the ruling.", @@ -336,9 +336,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "Tiers are compiled, pure test, convex-test and native backend, with composition and target recorded apart; acceptance check, verifier, test result and run record are defined; scenario is a row of the doc's table and a case is one example of it; CONTEXT.md carries them. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-010", @@ -681,7 +681,7 @@ "provisionalReading": "Extension E-5: the doc says a query authorizes before it discloses and does not say how a refused query is answered; here a parent query's refusal is the same `RejectionData` a command's is, with the query's name in the field `commandType` and only the codes `unauthenticated` and `forbidden`, and a use case's own rejection is thrown as the kernel's bare `Rejection`, the way a context throws one; that a field named for a command carries a query's name is a gap in the language, and the owner has not ruled on either reading (E-5, D7, Law 5)", "blocks": null, "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, a term of `CONTEXT.md`: `CONTEXT.md:55` defines rejection as a command's outcome and the query Specs use it for a query, and a field named for a command carries a query's name; the owner queue lists both as a gap in the language [design/STATE.md:236], a pointer. The shape and the two codes are mechanism, so the fork gives them as the lean.", @@ -690,9 +690,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "The shared RejectionData with the codes unauthenticated and forbidden for a query stays, a use case's own rejection stays bare, and the field is renamed entry, the name of the entry that refused; the code follows in the language lane. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-018", @@ -887,7 +887,7 @@ "provisionalReading": null, "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, it rules on terms of CONTEXT.md: no word for a parent's read, a context's read, the first activation or a refused function's name, rejection used for a query, projection without its per-entity name, three fixture names unlisted (design/STATE.md:236); the mechanism behind the query's name in commandType is OD-017 (domain) and OD-110 (convex), and only the word is here.", @@ -896,9 +896,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "Parent query, context query and query refusal are the words; the field that names the function that refused is entry, the name of the entry (public command, internal command, parent query or operator entry) that refused; installation includes the first rebuild; the fixture's names stay the fixture's. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-023", @@ -1184,7 +1184,7 @@ "provisionalReading": null, "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, it rules on terms of CONTEXT.md: operation, command, subject, actor and duplicate each against another word, and whether CONTEXT.md or spec:platform.vocabulary leads (design/STATE.md:243); receipt and generation are OD-139 and OD-140 and are not decided twice here.", @@ -1193,9 +1193,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "CONTEXT.md leads and spec:platform.vocabulary follows it; a command addresses the parent, a stream command a decider, an operation is one accepted execution; subject, actor and duplicate keep their meanings; the operator term splits into the operator actor, which nothing establishes, and the stated operator. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-030", @@ -1653,7 +1653,7 @@ "provisionalReading": null, "blocks": "S5", "advisor": "domain", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, goes against a decision of the doc: [design/specs/decisions/d03-events-only-source-of-next-state.sdp.md:35] says commands load current state and never replay history, and [design/specs/context/journal.sdp.md:45] lets `migrate` read the events since the last baseline when the adapter runs it on load.", @@ -1662,9 +1662,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "D3 holds on the command path: a migration runs on load only when every step derives the new state from the saved state alone; a history-reading step is written by the sweep alone, and a command or a read on a stream the sweep has not reached is refused as transient (writePaused), so no stream type is paused and no command reads a tail; the restore's journal check inventories awaitingBaseline as pending work, not a finding, and the accepted branch restarts the sweep. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-042", @@ -1684,7 +1684,7 @@ "provisionalReading": null, "blocks": "S5", "advisor": "operator", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Owner test 5: whether a failed check keeps every tenant's writers out with no override, and how often the drill runs, trade availability against safety and a standing operator duty against assurance, with no evidence on which weighs more, and the doc says none of them [design/reviews/consensus-ledger.json:2301]; it stands apart from OD-082, which is the mechanism and names these policies as outside it.", @@ -1695,9 +1695,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "A failed check keeps the whole deployment closed with no override in the first implementation; the drill runs before every release, its cadence reconsidered on the first drill's measured check times; both are E-42's, marked. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-043", @@ -1755,7 +1755,7 @@ "provisionalReading": null, "blocks": "L3", "advisor": "convex", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1: the doc says no deployment-wide order token exists [docs/convex-transactional-domain-platform-decisions.md:108] and the commit-timestamp page says one does; the doc against a Convex source resolves nothing without the owner, and D2's reason is the doc's text. Blocks stays L3, where D18's ordered consumer and the history view weigh it.", @@ -1769,9 +1769,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "D2's premise corrected: Convex documents a commit timestamp that orders committed transactions deployment-wide, which orders transactions and not the events of one and is no consumer protocol by itself; context-owned journals and stream-local order stay; the ordered consumer weighs the token with D18's consumer; F18 says the documented upper bound getSnapshotTs() is absent from the pinned package's public declarations; the doc's line 108 changes by the owner's hand. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-045", @@ -1890,16 +1890,16 @@ "provisionalReading": null, "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, a licence; docs/modern-ts.md:215 lists LICENSE under files and the root holds no such file, and package.json has no files field, so a publish today ships none; the lens records what a licence text says and leans on none.", "lean": null, "checkedBy": null, "fork": "design/decisions/forks/OD-048.md", - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "Apache-2.0, for its explicit contributor patent grant: LICENSE at the root and license in package.json; publication stays a separate decision. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-049", @@ -2473,7 +2473,7 @@ "provisionalReading": "OQ1: whether a trivial context with no invariants of its own may be plain tables in the parent behind lint rules instead of a component is decided after the experiment measures component call cost; the design assumes a component per context everywhere (OQ1, D2)", "blocks": null, "advisor": "convex", - "status": "waiting", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1: OQ1 is the doc's open question and D2 assumes a component per context [design/specs/context/context-component.sdp.md:49; design/PLAN.md:689]; answering it edits the doc.", @@ -2484,9 +2484,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "OQ1 answered: one component per context; reopened only for a measured workload that misses an agreed target or a hosted usage reading above an agreed share; Probe 3's quota half stays a separate measurement (OD-036). Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-064", @@ -4491,7 +4491,7 @@ "provisionalReading": null, "blocks": null, "advisor": "convex", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1: it rules on a term, whether the language gains a word for the function that refused or the field is renamed [design/specs/context/queries.sdp.md:37].", @@ -4500,9 +4500,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "The field is entry on the wire, one validator and one word; the rename of the code follows in the language lane. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-111", @@ -4685,7 +4685,7 @@ "provisionalReading": null, "blocks": "S5", "advisor": "operator", - "status": "waiting", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Owner test 1: if Probe 7 lets the do-nothing option stand for the durable profile, D13's chosen option changes, which edits the doc, and [design/specs/obligations/do-nothing-check.sdp.md:45] has the owner restate readiness after the probe.", @@ -4696,9 +4696,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "D13 stands: obligations stay the durable profile's choice, built at Layer 3's trigger; Probe 7's facts are recorded, and the do-nothing option fails Sc L3-4, L3-7 and L3-8. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-115", @@ -5743,7 +5743,7 @@ "provisionalReading": "OQ5 asks whether the decisions move into SDP carriers; this corpus is that move, with ledger facts as constraint Specs, decisions as decision Specs that list the do-nothing option first, and scenarios as example Specs under the component that provides the capability; the owner confirms the mapping or chooses rule Specs for facts, which is a mechanical change (OQ5)", "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, the doc's own question OQ5, which only the owner closes in the doc (design/specs/platform/transactional-domain-platform.sdp.md:44); doubt: the carrier choice itself is mechanical and would be tactical on its own.", @@ -5752,9 +5752,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "OQ5 confirmed: facts are constraint Specs, decisions are decision Specs, scenarios are example Specs under the component that provides the capability. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-139", @@ -5776,7 +5776,7 @@ "provisionalReading": "OQ6, receipt: this design uses receipt for the stored outcome of a command for idempotency, while today's glossary uses receipt for a row that proves an effect happened; the corpus reads the latter as completion evidence and the owner picks one meaning before code names it (OQ6, Vocabulary)", "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, the doc's own question OQ6 and a term of CONTEXT.md; the code already names receipt for the stored idempotency outcome (design/STATE.md:243, src/command/receipts.ts).", @@ -5785,9 +5785,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "OQ6, receipt: the stored outcome of a command; completion evidence proves an effect happened. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-140", @@ -5809,7 +5809,7 @@ "provisionalReading": "OQ6, generation: this design uses generation for a numbered build of a read model, while today's glossary uses it for the attempt number of watched work; the corpus reads the latter as attempt and the owner picks one meaning before code names it (OQ6, Vocabulary)", "blocks": null, "advisor": "product", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "owner", "reason": "Rule 1, the doc's own question OQ6 and a term of CONTEXT.md; the code already names generation for a numbered read-model build (design/STATE.md:243).", @@ -5818,9 +5818,9 @@ }, "checkedBy": null, "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "OQ6, generation: a numbered build of a read model; attempt is one try at an obligation. Ruled by the main thread on the owner's instruction of 2026-10-03 (\"make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra ... And rule.\"), from the recommendations in the unit folder after one Opus and one astra read, and recorded as the owner's as sitting one's were (decision 40)." }, { "id": "OD-141", diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index 373963f..20230c0 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -2274,10 +2274,11 @@ ], "issue": "The round-2 migrate-on-load mechanism overrules a sentence of D3 in one case and no Spec says so. The doc, D3: 'Commands load current state and apply only the new events; they never replay history.' persistence-adapter repeats it as a Contract bullet, 'Commands load current state and apply only the new events; the adapter never replays history on a command (D3)', and domain-kernel as a rule. In the same adapter Spec the [extension] step 3a calls `migrateToSchemaVersion` on load, and journal's `fnMigrateToSchemaVersion` says it will 'read the tail by `by_stream` above `baselineVersion ?? 0` when the step reads history, a tail above `maxFoldEvents` being a plain error, compute `step.migrate(saved, history)`'. journal's own risk bullet states the effect: 'A migration that reads history makes the first command on each of its streams read that stream's tail, at most `maxFoldEvents`'. So for a `readsHistory` migration a command reads and folds up to 600 past events before `decide`, which is a replay of history inside a command, and the adapter's D3 bullet is false as written. The E-25 open question lists the mechanism ('optionally the events since the last baseline', 'called by the adapter on load') but never names the departure from D3, and README's E-25 row does not mention the on-load path at all, so the owner is not asked to rule on it. The option that keeps D3 whole, history-reading migrations run by the sweep only, is not recorded as an alternative.", "fix": "On `spec:context.journal`, which owns E-25: either (a) restrict the on-load path to migrations with `readsHistory: false`, have a `readsHistory` migration run by the sweep only, under a closed `source:` scope for its stream type so no command meets an unmigrated stream, and say so in `migrateToSchemaVersionCallers`, `migrationWindow` and the adapter's step 3a; or (b) keep the on-load tail read and state in the E-25 question that it is a departure from D3's 'they never replay history', bounded by `maxFoldEvents` and limited to the first command per stream per meaning change, with (a) recorded as the alternative the owner may prefer. In both cases reword persistence-adapter's Contract bullet so it does not claim more than step 3a allows, and add the on-load migration to README's E-25 row.", - "status": "open", - "note": "", + "status": "fixed", + "note": "Ruled by OD-041: a history-reading migration is written by the sweep alone, and a command or read on a stream it has not reached is refused as transient; D3 holds on the command path.", "slice": "S5", - "detector": "judgement" + "detector": "judgement", + "unreviewed": true }, { "round": 3, @@ -2290,10 +2291,11 @@ ], "issue": "The journal both rejects and prescribes a write pause for a meaning migration, and the pause it prescribes stops the sweep it is meant to cover. The narrative says 'No decision is therefore taken against a state of the wrong meaning, and nothing pauses writes', and the E-25 question says refusing commands on unmigrated streams 'was not taken because it turns a deploy into a partial write pause the doc reserves for history-dependent views' (D9: the write pause 'is the exception'). The fourth risk bullet then says 'a meaning change is therefore rolled out in a maintenance window or under the tenant's write pause, and the driver reads the gate before every batch', and write-pause's `fnCloseGate` says the operator's close of 'a tenant, a source stream type or the `all` scope, with no generation' is 'also how a meaning migration's sweep is run under a pause when the operator wants one'. But `baselineDriver` reads `gateAllows(ctx, [\"all\", \"tenant:\" + tenantId, \"source:\" + contextId + \":\" + streamType])` with no generation and 'when the answer is not allowed it reschedules itself after `limitDriverDeferral` with the same cursor and writes nothing', and `limitDriverDeferral` says 'a closed tenant, `all` or source scope delays the sweep and never ends it'. `gateAllows` lets a batch through only when the closed entry's `generationId` equals `allowGeneration`, which the driver never passes. So a sweep cannot run under the tenant's pause: it waits until the pause is lifted. An implementer cannot build the rollout the risk bullet prescribes, and the reader cannot tell whether the design pauses writes for a meaning change (a use of the pause beyond D9) or not. Related to r3-architecture-reads-of-streams-awaiting-baseline and r3-architecture-restore-and-baseline-migration-disagree, which are about reads and restore in the same window; this is the write side.", "fix": "Rule one way on `spec:context.journal` under E-25 and mirror it in `spec:application.write-pause` under E-8. Either the sweep never runs under a pause: delete 'or under the tenant's write pause' from the risk bullet and the clause in `fnCloseGate`, and state that the version bump is a cost callers absorb by re-review. Or the sweep may run under an operator-closed scope: give the driver an admission the gate can check (an operator-closed entry carries the migration name, and `gateAllows` takes it the way it takes `allowGeneration`), state in the E-25 and E-8 questions that this uses the write pause for something D9 does not name, and reconcile the narrative's 'nothing pauses writes'.", - "status": "open", - "note": "the platform review of 2026-10-02, CORE-06: At HEAD, journal.sdp.md:40 prescribes \"under the tenant's write pause\", :109 reschedules without writes when the gate is closed, and :112 says \"no writer is paused\", so the prescribed paused sweep cannot run. The issue and one ruled admission policy remain right; cold read models must also reach the migrated meaning. The build must show: Native backend migration must hold a tenant or source write pause and demonstrate exactly the ruled admission policy: ordinary writers remain refused and a permitted migration either advances safely or remains deferred until reopening.", + "status": "fixed", + "note": "the platform review of 2026-10-02, CORE-06: At HEAD, journal.sdp.md:40 prescribes \"under the tenant's write pause\", :109 reschedules without writes when the gate is closed, and :112 says \"no writer is paused\", so the prescribed paused sweep cannot run. The issue and one ruled admission policy remain right; cold read models must also reach the migrated meaning. The build must show: Native backend migration must hold a tenant or source write pause and demonstrate exactly the ruled admission policy: ordinary writers remain refused and a permitted migration either advances safely or remains deferred until reopening. Ruled by OD-041: no migration runs under a pause; a state-only one pauses nothing, and a history-reading one holds only the streams the sweep has not reached. The journal's rollout sentence and the gate's clause are cut.", "slice": "S5", - "detector": "convex-test" + "detector": "convex-test", + "unreviewed": true }, { "round": 3, @@ -2306,10 +2308,11 @@ ], "issue": "Three operator policies of the restore are stated as the doc's, and the doc says none of them. (1) 'A restore that fails a check stays closed, and the run record names the failed check and its subject; nobody reopens a deployment that failed a check (Sc L2-8, D19)', and the outcome line 'reopens only on a clean result (D19, Sc L2-8)'. Sc L2-8's pass condition is 'Domain, journal and read-model invariants hold; the procedure runs', and D19 says only 'Restore starts with external dispatch off, reconciles the gap since the backup, and rebuilds schedules from obligations' and 'restore is tested, not only backup'. Neither says writers stay out until four checks are clean, nor that a single finding in one tenant's stream keeps every tenant's writes refused with no operator override. That is an availability ruling for the worst day of a deployment and it belongs to the owner. (2) 'the drill runs this workflow on a representative dataset in a disposable deployment before every release (D19, Sc L2-8)' and `drillCadence: once per release`. The doc sets no cadence; a drill per release is a standing operator duty, which decision method rule 2 says a mechanism must state as its cost. (3) 'Restore starts with dispatch off; in the transactional profile that is the writer door closed through the environment until the checks pass (D19, E-42)' drops the doc's word 'external' and reads dispatch off as writers off; the bullet has no [extension] mark. The E-42 open question lists the four checks, the inventory, the `restoreRuns` record, the accepted branch and 'the `MAINTENANCE_MODE` environment variable as the door', but not that a failed check blocks reopening, not the cadence, and not the baseline-chain rule that the third [extension] rule cites under E-42. So the owner cannot confirm or reject these from the register. Related: r3-architecture-restore-and-baseline-migration-disagree shows a case where the fail-closed rule cannot be satisfied.", "fix": "On `spec:application.restore`, which owns E-42: mark rules (1), (2) and (3) [extension] under E-42 and remove the D19 and Sc L2-8 citations from the parts the doc does not say; add to the E-42 question that the design keeps every writer out until all four checks are clean with no override, the alternative being an operator-acknowledged reopen with findings recorded (or a reopen per tenant or per check class), that the drill cadence is once per release and is an operator duty, and that a baseline chain in flight is restarted by the operator; add the same three points to README's E-42 row. State the drill's cost in a risk bullet.", - "status": "open", - "note": "", + "status": "fixed", + "note": "Ruled by OD-042: the three policies are E-42's, marked, and the owner confirmed fail closed with no override and the drill before every release.", "slice": "S5", - "detector": "judgement" + "detector": "judgement", + "unreviewed": true }, { "round": 3, @@ -3026,7 +3029,7 @@ "issue": "A parent query's refusal has no pinned wire shape, and the one pinned example fits no contract. queries `parentPassThrough` throws `new ConvexError({ code: \"unauthenticated\", message: \"sign in\" })` and `new ConvexError({ code: \"forbidden\", message: decision.reason })`. outcome-boundary says 'A rejection is thrown as `new ConvexError(data)` with `data.kind` equal to `\"rejection\"`' and that `errorDataValidator` 'describes it exactly'; that validator requires `kind` and `commandType`, which a query has none of, and `normalizeThrown` runs only in the pipeline handler. A client that tells a rejection by `data.kind` cannot classify a query's refusal, and `errorCodeUnauthenticated` and `errorCodeForbidden` name only the pipeline's steps 2 and 4 as throwers. read-models says only that the query 'returns nothing before authorization passes'. The operator operations have the same gap: operator-operations says an operation on a terminal obligation 'is refused with a rejection through the outcome boundary' and pins `errorCodeObligationNotRepairable`, `errorCodeLeaseInFlight`, `errorCodeEvidenceRequired` and `errorCodeReconcileUnsupported`, but these functions are 'not domain commands', have no declaration whose `rejections` could name the codes, and outcome-boundary says 'a code outside both is a technical failure by definition'.", "fix": "Rule once in `spec:command.outcome-boundary`: either a parent query throws through `reject` with `commandType` set to the query's name (and the field is renamed or documented as the function name), or the rejection branch's `commandType` becomes optional and absent for queries. Then make `parentPassThrough` in `spec:context.queries` throw through that helper and add the query path to the two `errorCode` bullets. Apply the same ruling to the operator operations of `spec:obligations.operator-operations`, and say where their four codes are closed.", "status": "partially-fixed", - "note": "S1, 2026-10-01. Slice S1 built no query (ruling R2); `normalizeThrown` now takes the declaration's `rejections`, so a code no declaration lists is a technical failure on the command path. S2, which builds queries, must rule the wire shape of a parent query's refusal in `spec:command.outcome-boundary` and show in a test that a client classifies it by `data.kind`; the operator operations follow in S4. S2, 2026-10-02, fold-in after the build. A parent query's refusal is `RejectionData` with the query's name in `commandType` and only the codes `unauthenticated` and `forbidden` (`spec:command.outcome-boundary`), shown at the native tier on the production composition by `tests/native/read-models.query-refused-before-disclosure.test.ts`, an HTTP call and a subscription, and `tests/native/read-models.query-refused-without-grant.test.ts`, and on the fixture composition by `tests/native/read-models.test.ts`. Left for L3, which builds them: the operator operations of `spec:obligations.operator-operations`; and for the owner, the language gap of a field named for a command that holds a query's name.", + "note": "S1, 2026-10-01. Slice S1 built no query (ruling R2); `normalizeThrown` now takes the declaration's `rejections`, so a code no declaration lists is a technical failure on the command path. S2, which builds queries, must rule the wire shape of a parent query's refusal in `spec:command.outcome-boundary` and show in a test that a client classifies it by `data.kind`; the operator operations follow in S4. S2, 2026-10-02, fold-in after the build. A parent query's refusal is `RejectionData` with the query's name in `commandType` and only the codes `unauthenticated` and `forbidden` (`spec:command.outcome-boundary`), shown at the native tier on the production composition by `tests/native/read-models.query-refused-before-disclosure.test.ts`, an HTTP call and a subscription, and `tests/native/read-models.query-refused-without-grant.test.ts`, and on the fixture composition by `tests/native/read-models.test.ts`. Left for L3, which builds them: the operator operations of `spec:obligations.operator-operations`; and for the owner, the language gap of a field named for a command that holds a query's name. Sitting two ruled the shared shape, the two query codes, the bare use-case rejection and the field `entry`; the operator-operation codes stay the open half.", "slice": "L3", "detector": "pure unit test" }, @@ -3750,8 +3753,8 @@ ], "issue": "d02's Decision context states as a Convex fact: 'Convex gives components whose tables the parent cannot read except through the component's own functions, and gives no global sequence, so a global position would have to be a counter document in every write's path'. Current Convex gives one. 'A commit timestamp represents the time (in nanoseconds) when a mutation commits ... with the `db.vars.commitTs` placeholder you can have the value inserted at commit time into documents'; 'You could define an index on a field that captures the commit time, then use that to iterate over new or updated documents without worrying about missing changes due to out-of-order commits'; the comparison table lists 'Reflects commit order: Yes' against `_creationTime` 'No'; 'the timestamp is strictly increasing' (docs.convex.dev/database/advanced/commit-timestamp, read 2026-10-01). `ctx.meta.getSnapshotTs()` gives the matching upper bound: 'no new documents will be written with a `commitTs` at or below this' (docs.convex.dev/api/interfaces/server.QueryMeta, read 2026-10-01). So an order token that costs no counter document and no contention exists, and it does prove that a reader has seen every committed row up to the snapshot. The doc's own rule, 'Timestamps, `_creationTime` and UUID order never prove that a consumer has seen every committed event', is still right for `_creationTime` and `recordedAt` and is not right for a commit timestamp. Nothing fails, and the doc's decision stands until the owner rules. But the reason d02 gives for rejecting a global position is no longer a fact, the ordered consumer of D18 pays 'a write to every command that produces a selected event' for a sequence the engine can now supply, and the history view's cross-stream order problem (r3-architecture-history-fold-order-and-deletion) has a native answer the corpus does not weigh.", "fix": "In `spec:decisions.d02-context-owns-state-and-journal` correct the context sentence: Convex gives no contiguous global sequence, and gives a commit timestamp that reflects commit order. Add the native option to the alternatives as a do-nothing-first entry with what it does and does not give: order by commit across the tables of one component, no gap detection by arithmetic, a nanosecond `Int64` that cannot be scheduled as an argument, one clock per deployment. Add a `[non-blocking]` open question for the owner on d02, on `spec:advanced.ordered-consumer` and on the trigger table's global feed row: whether an indexed `v.commitTs()` field on the events table replaces the allocated consumer sequence. Change no mechanism until the owner rules.", - "status": "open", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: An ordered consumer is about to add a contiguous counter although current Convex offers commit-order timestamps and a snapshot upper bound; that alternative supplies order but not arithmetic gap semantics, immutable historical state or cross-context business meaning. The build must show: Native backend probes on the pinned release must test ordered scans bounded by the snapshot through concurrent commits and interruptions, and state which gap and rebuild guarantees remain unsolved. Only the owner may change D2 or the consumer mechanism after weighing that evidence.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: An ordered consumer is about to add a contiguous counter although current Convex offers commit-order timestamps and a snapshot upper bound; that alternative supplies order but not arithmetic gap semantics, immutable historical state or cross-context business meaning. The build must show: Native backend probes on the pinned release must test ordered scans bounded by the snapshot through concurrent commits and interruptions, and state which gap and rebuild guarantees remain unsolved. Only the owner may change D2 or the consumer mechanism after weighing that evidence. OD-044 corrected D2's premise and F18's wording; the consumer's use of the token is weighed with D18's consumer.", "slice": "L3", "detector": "judgement" }, @@ -4325,11 +4328,11 @@ ], "issue": "The foundation vocabulary calls a command a request to one context and gives a context one decider. CONTEXT.md instead separates command from stream command and assigns a decider to a stream type. The Spec also prohibits receipt and generation names already used in code.", "fix": "Obtain the queued owner ruling on which vocabulary leads, then align the command, stream command, decider, receipt and generation entries and the permitted provisional language.", - "status": "open", - "note": "the platform review of 2026-10-02, FOUND-02. Evidence recorded by that read: Spec and source inspection. Counterexample: A builder following Pack modelRefs gets a command for one context and a context-wide decider, while CONTEXT.md gives a parent command, a stream command and a decider per stream type; the Spec also forbids receipt and generation names already in code. Spec evidence at 2e20b29: No ledger match. platform/vocabulary.sdp.md:37,38 conflicts with CONTEXT.md; :21,25,26 prohibit names already used. STATE.md:183 carries the authority question.", + "status": "fixed", + "note": "the platform review of 2026-10-02, FOUND-02. Evidence recorded by that read: Spec and source inspection. Counterexample: A builder following Pack modelRefs gets a command for one context and a context-wide decider, while CONTEXT.md gives a parent command, a stream command and a decider per stream type; the Spec also forbids receipt and generation names already in code. Spec evidence at 2e20b29: No ledger match. platform/vocabulary.sdp.md:37,38 conflicts with CONTEXT.md; :21,25,26 prohibit names already used. STATE.md:183 carries the authority question. Ruled at sitting two: CONTEXT.md leads, and the vocabulary Spec's command, stream command, decider, receipt and generation entries follow it.", "slice": "P", "detector": "Source review after the owner's authority ruling must find one meaning for command, stream command, decider, receipt and generation and one permitted provisional naming policy. Compiled declarations using those terms must remain valid; compilation alone cannot prove vocabulary agreement.", - "unreviewed": false + "unreviewed": true }, { "round": "PR1", diff --git a/design/specs/advanced/ordered-consumer.sdp.md b/design/specs/advanced/ordered-consumer.sdp.md index fce1397..be2a076 100644 --- a/design/specs/advanced/ordered-consumer.sdp.md +++ b/design/specs/advanced/ordered-consumer.sdp.md @@ -8,6 +8,7 @@ relations: constrainedBy: - spec:laws.law10-replay-never-runs-commands-or-effects - spec:facts.f01-serializable-mutations-under-occ + - spec:facts.f18-commit-timestamps --- # Ordered event consumer @@ -36,7 +37,7 @@ Order is per tenant, consumer, generation and partition, with the smallest parti - rule: A duplicate delivery is applied once because the consumer position advances only past applied sequence numbers (D18, F1) - rule: A poison event blocks only its partition; skipping it is an authorized, recorded data-loss decision, and the consumer contract says whether it is allowed (D18) - rule: The consumer never runs commands or external effects while applying events (D18, Law 10) -- rule: No deployment-wide order token exists; a global ordered feed is a separate capability with its own trigger (D18, D2) +- rule: The journal keeps no deployment-wide order token; a global ordered feed is a separate capability with its own trigger, and the engine's commit timestamp of F18 is weighed as its position when that consumer is designed (D18, D2, F18) ## Design diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index a3acd76..8a88c4a 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -56,16 +56,16 @@ The six cost targets are the constraint Specs this workflow is constrained by. T ### Open questions -- [non-blocking] OQ1: whether a trivial context with no invariants of its own may be plain tables behind lint rules is decided after this experiment measures component call cost (OQ1, Probe 3) +- [non-blocking] OQ1: a trivial context with no invariants of its own stays a component; the owner answered OQ1 on this experiment's local measurement of component call cost, and a measured workload that misses an agreed target, or a hosted usage reading above an agreed share, reopens it (OQ1, Probe 3) - [non-blocking] Extension E-46: the doc asks for one more lifecycle command and one essential summary without naming them; the option taken here is `CancelOrder`, which releases the allocation, and the order summary read model (E-46, First experiment) - [non-blocking] Extension E-47: the doc lists what to count but not how; the option taken here is a measurement record per run captured from the disposable backend's function execution log, documents and bytes from the `usageStats` of the command's top-level completion record, and the calls per context from a pure test of the use case's executor, because the log holds no record of a component call inside a mutation, with any count the backend does not expose recorded as a gap; the owner has not ruled (E-47, First experiment) - [non-blocking] Extension E-47, the documents a command reads: the design expected one `PlaceOrder` of N lines to read about N + 6 documents, and the pinned backend counted 3N + 5; the measurement run decides where the other 2N reads come from (E-47, F13) - [non-blocking] Extensions E-8, E-13, E-15 and E-37, as the experiment's setup takes them: stock exists only through `ReceiveStock`, a run seeds its grants and makes the order summary writable with admin access before its first command, the production composition names no service issuer, its `auth.config.ts` reads its issuer from environment variables, and it is the Convex app whose project directory is `example/`; these are options taken here and the owner has not ruled (E-8, E-13, E-15, E-37, E-46) -- [non-blocking] Extension E-17, the rows the experiment requires: the doc says the experiment passes when every Layer 0, 1 and 2 scenario passes, and its one all-layer row, Sc ALL-1, applies to every installed layer; the option taken here requires the twenty-three rows the doc's table gives Layer 0, 1 or 2 and leaves Sc ALL-1 to the Spec that owns it; the owner rules whether the experiment's pass includes Sc ALL-1 (E-17, First experiment, Sc ALL-1) +- [non-blocking] Extension E-17, the rows the experiment requires: the doc says the experiment passes when every Layer 0, 1 and 2 scenario passes, and its one all-layer row, Sc ALL-1, applies to every installed layer; the option taken here requires the twenty-three rows the doc's table gives Layer 0, 1 or 2 and Sc ALL-1, each at the tier the acceptance contract names for it, so that the native proof of the audit and the diagnostic is part of the experiment; the owner confirmed both (E-17, First experiment, Sc ALL-1) ## Workflow -- rule: The experiment passes when every Layer 0, 1 and 2 scenario passes on a native backend (First experiment) +- rule: The experiment passes when every Layer 0, 1 and 2 scenario and Sc ALL-1 passes, each at the tier `spec:platform.acceptance-contract` names for it (First experiment, Sc ALL-1) - rule: [extension] Whether the experiment passed is answered by the acceptance check of `spec:platform.acceptance-contract`, over the rows `acceptanceRows` names and one run's record; no single example and no single test answers it (E-17, First experiment) - rule: It builds Layers 0 to 2 in a small, clean application: Orders and Inventory, complete order placement, one more lifecycle command, one essential summary, journal inspection and rebuild (First experiment) - rule: It reuses the pure deciders and state machines where they fit, under the one-authority rule of D3, and leaves the current app's infrastructure behind (First experiment, D3) @@ -132,7 +132,7 @@ The experiment is an application, a test suite and a report. The application is - targetContention: with N `PlaceOrder` commands at once on one stock item that has stock for one, at N of 2 and 8 every command ends applied or rejected, one applied and the rest rejected for short stock, and the engine fails none after its bounded retries; at N of 32 the same count is reported, and a command the engine failed is a recorded cost and no failure of the experiment, because the caller repeats it under the same request key (First experiment, Sc L1-11, F1) - deliverables: the application in this repository, beside the design, which settles OQ4, the native evidence records of every scenario run, and the measurement report with the six cost targets and the speed targets marked held or not held against measured fields; one explicit timing command records the latency distribution after discarded warmups and throughput with its write cap, prints each fixed speed target as held or not held, and labels readings taken under load for replacement by a quiet-machine reading; speed misses are report values, not failing assertions (First experiment, OQ4, Acceptance scenarios, E-47) - authProvider: [extension] the production composition's `auth.config.ts` declares one `customJwt` provider whose `issuer`, `applicationID` and `jwks` are environment variables of the deployment, as `spec:platform.native-harness` pins for a composition that runs natively (E-13, D11) -- acceptanceRows: [extension] the rows the experiment's acceptance check requires are Sc L0-1, Sc L0-2, Sc L1-1, Sc L1-2, Sc L1-3, Sc L1-4, Sc L1-5, Sc L1-6, Sc L1-7, Sc L1-8, Sc L1-9, Sc L1-10, Sc L1-11, Sc L1-12, Sc L2-1, Sc L2-2, Sc L2-3, Sc L2-4, Sc L2-5, Sc L2-6, Sc L2-7, Sc L2-8 and Sc L2-9, which are the rows the doc's table gives Layer 0, 1 or 2 (E-17, First experiment, Acceptance scenarios) +- acceptanceRows: [extension] the rows the experiment's acceptance check requires are Sc L0-1, Sc L0-2, Sc L1-1, Sc L1-2, Sc L1-3, Sc L1-4, Sc L1-5, Sc L1-6, Sc L1-7, Sc L1-8, Sc L1-9, Sc L1-10, Sc L1-11, Sc L1-12, Sc L2-1, Sc L2-2, Sc L2-3, Sc L2-4, Sc L2-5, Sc L2-6, Sc L2-7, Sc L2-8, Sc L2-9 and Sc ALL-1, which are the rows the doc's table gives Layer 0, 1 or 2 and the one all-layer row (E-17, First experiment, Acceptance scenarios) ## Example space @@ -158,6 +158,6 @@ And the order's OrderPlaced payload measures {payloadBytes:number} bytes ## Verification — reviewed -- The acceptance check confirms that every row `acceptanceRows` names has an example with a verifier and a passing result in one run's record, and a pure test confirms that `acceptanceRows` names exactly the rows the doc's table gives Layer 0, 1 or 2. +- The acceptance check confirms that every row `acceptanceRows` names has an example with a verifier and a passing result in one run's record, and a pure test confirms that `acceptanceRows` names exactly the rows the doc's table gives Layer 0, 1 or 2 and Sc ALL-1. - A reviewer confirms that each of the six constraint targets is compared against a measurement field and that the comparison is in the report. - A reviewer confirms that the reused deciders return events and a result only, never a state patch. diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index 60eddb5..bab3e3f 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -48,7 +48,7 @@ This Spec carries the read-need table as rules, the properties of a projection, - [non-blocking] Probe 5 showed that the built-in `.paginate` treats `maximumRowsRead` as advice once a page has an end cursor and that a cursor it issued is refused by any other query, so a subscription over the active generation threw `InvalidCursor` at a switch; a parent list therefore pages with `paginator` of `convex-helpers`, as a context list does, and a pinned page is bounded on every run by the row cap and by the byte cap as `spec:context.queries` states it, the helper stopping at the row that reaches it and admitting that row whole, and answers `SplitRequired` beyond them (Probe 5, F13, D8, D9) - [non-blocking] Extension E-24, as it reaches read models: a list over a parent read-model table calls `paginator` of `convex-helpers` with the options of `boundedPage`, a row cap of twice the item cap, with each cursor of the client's pair moved into the active generation by `pageInGeneration`, and returns through `paginationResultValidator` with its rows mapped by `readModelView`, and a client pages every list by an explicit cursor pair with no hook; these are options taken here and the owner has not ruled (E-24, S4, Probe 5, D8) -- [non-blocking] Extension E-6, as it reaches queries: a parent query establishes and authorizes its actor through `authorizeQuery`, and its refusal is the command's rejection shape, `RejectionData` with the query's name in the field `commandType` and the codes `unauthenticated` or `forbidden`; the field keeps its name for a query, which the owner has not ruled on, and nothing else of the shape is new (E-6, D7, D11, Law 5) +- [non-blocking] Extension E-6, as it reaches queries: a parent query establishes and authorizes its actor through `authorizeQuery`, and its refusal is the command's rejection shape, `RejectionData` with the query's name in the field `entry` and the codes `unauthenticated` or `forbidden`; the owner confirmed the word, and nothing else of the shape is new (E-6, D7, D11, Law 5) - [non-blocking] Probe 3 pending: the cost of one component call in latency and function-call quota decides whether one entity's detail stays a component query or hot reads move, which is OQ1 (Probe 3, F4, OQ1) - [non-blocking] Extension E-45: the doc says a small cross-context view is a parent query over bounded component reads and does not give the bound; the option taken here is at most 8 component reads per parent query and at most 4 read-model rows written per command, both measured by the first experiment (E-45, D8) diff --git a/design/specs/application/restore.sdp.md b/design/specs/application/restore.sdp.md index 21f2b8d..706ac36 100644 --- a/design/specs/application/restore.sdp.md +++ b/design/specs/application/restore.sdp.md @@ -53,22 +53,22 @@ The parts of restore that concern obligations, external dispatch and provider ev ### Open questions - [non-blocking] Probe 7 observes local CLI replacement restoring the parent, Orders and Inventory documents with IDs and creation times; fresh replacement leaves scheduler tables empty and replacement in place keeps existing schedules; the complete Layer 2 restore drill, hosted dashboard restore, Workpool and Workflow remain open (Probe 7, F12, D19) -- [non-blocking] Extension E-42: the doc says restore runs with matching code and configuration, that domain, journal and read-model invariants hold, and that restore is tested; it does not list the checks or the record, and it does not say what a restore does to a rebuild that was in flight at the backup. The option taken here: four checks, journal, domain, read model and receipt, each a bounded checkpointed batch, the stream check bounded through the `rebuild` query's `maxEvents` argument so a batch folds at most 500 events in all, each folded stream also reading its own stored baseline event, a fifth step that inventories the closed gate entries, the generations in `building`, `verifying` or `verified` and the `imports` rows in `running` that the snapshot restored, a `restoreRuns` row as the record, an accepted branch that resumes every stranded generation the operator did not abort, and the `MAINTENANCE_MODE` environment variable as the door (E-42, D19, F12) +- [non-blocking] Extension E-42: the doc says restore runs with matching code and configuration, that domain, journal and read-model invariants hold, and that restore is tested; it does not list the checks or the record, and it does not say what a restore does to a rebuild that was in flight at the backup. The option taken here: four checks, journal, domain, read model and receipt, each a bounded checkpointed batch, the stream check bounded through the `rebuild` query's `maxEvents` argument so a batch folds at most 500 events in all, each folded stream also reading its own stored baseline event, a fifth step that inventories the closed gate entries, the generations in `building`, `verifying` or `verified` and the `imports` rows in `running` that the snapshot restored, a `restoreRuns` row as the record, an accepted branch that resumes every stranded generation the operator did not abort, the `MAINTENANCE_MODE` environment variable as the door, a failed check keeping the whole deployment closed with no override, and the drill before every release with its cadence reconsidered on the first drill's measured check times; the owner confirmed the checks, the door and both policies (E-42, D19, F12) - [non-blocking] Extension E-42, the stream check's batch size: `limitCheckBatch` states the read of a batch that folds one stream, and each further stream the same batch folds reads its own stored baseline event of at most `budgetBytes + 4,096 + 61` bytes; the batch size that keeps a batch of several folded streams under the 16 MiB read ceiling is not pinned (E-42, F13) ## Workflow - rule: Restore runs with the code and configuration the backup was taken under; the recovery plan names code, schemas, configuration, credentials, data and external outcome evidence (D19, Sc L2-8) -- rule: Restore is tested, not only backup; the drill runs this workflow on a representative dataset in a disposable deployment before every release (D19, Sc L2-8) +- rule: [extension] Restore is tested, not only backup; the drill runs this workflow on a representative dataset in a disposable deployment before every release, a cadence reconsidered on the first drill's measured check times (E-42, D19, Sc L2-8) - rule: A backup excludes code, configuration, environment variables and scheduler rows; local CLI replacement replaces tables named in the archive, preserves destination environment variables, leaves a fresh scheduler empty and preserves an in-place destination scheduler; hosted dashboard restore is a separate unprobed path (F12, S8) -- rule: Restore starts with dispatch off; in the transactional profile that is the writer door closed through the environment until the checks pass (D19, E-42) +- rule: [extension] Restore starts with external dispatch off; in the transactional profile that is the writer door closed through the environment until the checks pass (E-42, D19) - rule: The checks never run a command, never repeat an external effect and never rewrite an event (Law 10, D5) - rule: Every check is a bounded, checkpointed batch below the transaction limits (D19, F13) - rule: After a restore the receipts in the snapshot are the authority for what committed; anything committed after the backup time is outside the restored state, and the run record states the backup time so callers can re-send with their request keys (D19, D6) -- rule: A restore that fails a check stays closed, and the run record names the failed check and its subject; nobody reopens a deployment that failed a check (Sc L2-8, D19) +- rule: [extension] A restore that fails a check keeps the whole deployment closed, with no override, and the run record names the failed check and its subject; nobody reopens a deployment that failed a check (E-42, Sc L2-8, D19) - rule: [extension] The four checks are journal, domain, read model and receipt, in that order, and each records what it examined and what it found on the run record (E-42, D19) - rule: [extension] A restore drops every pending scheduled function with the snapshot, so a generation in `building` or `verifying` at backup time comes back with no batch driving it and a closed gate entry that names it stays closed; the fifth step inventories both, plus every `imports` row in `running`, on the run record, and the accepted branch resumes every inventoried generation the operator did not abort, so the gate never stays closed for a rebuild nobody is running (E-42, F12, D19) -- rule: [extension] A baseline-migration chain in flight at the backup leaves no row, only the baseline events it had written; the operator restarts it without a cursor after the accepted branch, which is safe because the journal's `writeBaseline` skips every stream already at the target schema version (E-42, F12, E-25, D5) +- rule: [extension] A baseline-migration chain in flight at the backup leaves no row, only the baseline events it had written; the journal check inventories each stream it reports `awaitingBaseline` as pending work and not a finding, and the operator restarts the chain without a cursor after the accepted branch, which is safe because the journal's `writeBaseline` skips every stream already at the target schema version (E-42, F12, E-25, D5) - The operator picks the backup and the recovery-plan entry recorded with it: commit, schema hash, configuration hash, credential locations and the location of external outcome evidence (D19, Sc L2-8) - The operator sets `MAINTENANCE_MODE` to `restore` on the target deployment, so the writer check refuses every write with `writePaused` from now on (E-42, D19) - The operator deploys the recorded commit and configuration and confirms that the deployed schema matches the backup's schema hash (D19, Sc L2-8) @@ -109,7 +109,7 @@ The restore itself is a platform operation outside any mutation. Everything the - limitFoldPerStream: [extension] a fold check reads the events since the latest baseline, at most the journal's `maxFoldEvents` of 500 and never more than the batch's remaining `maxEvents`; the `rebuild` query reads the stream row and refuses before reading an event, with `tailTooLong` above the fold bound, which is a finding of the journal check and not a bigger batch, and with `budgetExhausted` above the batch's remaining budget, which is the resume point of the next batch and never a finding (E-42, E-25, F13, D5) - limitPendingWorkBatch: 100 `imports` rows per inventory call, one gate document and at most three registry index ranges per declared read model on the first call, all far below the ceilings; a generation in `building`, `verifying` or `verified` is at most one per read model, so the registry part is bounded by the number of declared read models (E-42, F13) - limitFindingsKept: at most 100 findings per check on the run record; further findings are counted, so the record stays under the document limit, and at most 200 pending-work items are kept the same way (E-42, F13) -- drillCadence: once per release on a disposable deployment, with the representative dataset the acceptance contract names (D19, Sc L2-8) +- drillCadence: [extension] once per release on a disposable deployment, with the representative dataset the acceptance contract names, reconsidered on the first drill's measured check times (E-42, D19, Sc L2-8) ## Example space diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index cfca537..c799b3a 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -70,7 +70,7 @@ The gate is one document so that a command pays one point read. Scopes are strin - fnScopesOfUseCase: `scopesOfUseCase(tenantId: string, writes: readonly SourceRef[]): ScopeKey[]` returning `"all"`, the tenant scope and one source scope per `SourceRef` of the declaration's `writes`, in that order, so a command is held by a closed deployment, by its closed tenant and by a paused rebuild of any view whose sources it writes (E-8, E-38, D9) - fnCloseScope: `closeScope(ctx: MutationCtx, scopeKey: ScopeKey, reason: string, operator: string, generationId?: Id<"generations">): Promise` throwing `"The scope " + scopeKey + " is already closed"` when `closed` holds an entry with that `scopeKey` and `"The gate holds 16 closed scopes, its limit"` when it holds `limitClosedScopes` entries; otherwise appending the entry to `closed`, inserting the document when there is none, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { kind: "gate.close", scopeKey, reason, generationId, operator })` in the caller's mutation; called by `closeGate` and by the rebuild's `startGeneration`, which passes the ID of the generation row it has just inserted (E-8, E-43, E-48, D9, D19) - fnResumeScope: `resumeScope(ctx: MutationCtx, scopeKey: ScopeKey, operator: string): Promise` throwing `"The scope " + scopeKey + " is not closed"` when `closed` holds no entry with that `scopeKey`; otherwise removing the entry, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { kind: "gate.resume", scopeKey, reason: entry.reason, generationId: entry.generationId, operator })` in the caller's mutation, the reason and the generation being those of the entry it removed; called by `resumeGate` and by the rebuild's `abortGeneration` and `switchGeneration` (E-8, E-43, E-48, D9, D19) -- fnCloseGate: `export const closeGate = internalMutation({ args: { scopeKey: v.string(), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `closeScope` with no generation: it calls `assertOperator`, throws `"closeGate takes the scope all or a tenant scope, and " + scopeKey + " is neither"` unless `scopeKey` is `all` or `tenant:` followed by a tenant ID of 1 to `limitIdLength` bytes, throws `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and then calls `closeScope`; it does not read the tenant list, so a tenant may be closed before its first grant; this is also how a meaning migration's sweep is run under a pause (E-8, E-25, E-48, D9, D19) +- fnCloseGate: `export const closeGate = internalMutation({ args: { scopeKey: v.string(), reason: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `closeScope` with no generation: it calls `assertOperator`, throws `"closeGate takes the scope all or a tenant scope, and " + scopeKey + " is neither"` unless `scopeKey` is `all` or `tenant:` followed by a tenant ID of 1 to `limitIdLength` bytes, throws `"The reason must be between 1 and 256 bytes"` unless `utf8Length(reason)` is, and then calls `closeScope`; it does not read the tenant list, so a tenant may be closed before its first grant (E-8, E-48, D9, D19) - fnResumeGate: `export const resumeGate = internalMutation({ args: { scopeKey: v.string(), operator: v.string() }, returns: v.null(), handler })` the operator entry to `resumeScope`: it calls `assertOperator`, makes the scope check of `closeGate` with `resumeGate` in the message, so a source scope is never reopened ahead of its rebuild, and then calls `resumeScope` (E-8, E-48, D9, Sc L2-6) - fnGetGate: `export const getGate = internalQuery({ args: {}, returns: v.object({ restore: v.boolean(), closed: v.array(closedEntryValidator) }), handler })` answering `restore` true when `MAINTENANCE_MODE` is `restore` and `closed` as the document holds it, empty when there is no document (E-8, E-42, D19) - fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page, a request for more answered with a page of 100, passing the paginator only the request's `cursor` and the capped `numItems`, so an `endCursor` in the request is ignored and cannot widen a page, where `operatorAuditDoc` is the `operatorAudit` table's document validator with `_id` and `_creationTime` (E-8, E-43, D19) diff --git a/design/specs/command/outcome-boundary.sdp.md b/design/specs/command/outcome-boundary.sdp.md index cc447f5..6355c35 100644 --- a/design/specs/command/outcome-boundary.sdp.md +++ b/design/specs/command/outcome-boundary.sdp.md @@ -36,7 +36,7 @@ The closed code list has two halves. The platform codes below are closed here an ### Open questions - [non-blocking] Extension E-5: the doc fixes the four outcomes and that a rejection is a structured `ConvexError`, but not the data shape or the code list; the shape and the platform codes below are the design's, and the domain codes are closed per declaration (E-5, D4, D7) -- [non-blocking] Extension E-5: the doc says a query authorizes before it discloses and does not say how a refused query is answered; here a parent query's refusal is the same `RejectionData` a command's is, with the query's name in the field `commandType` and only the codes `unauthenticated` and `forbidden`, and a use case's own rejection is thrown as the kernel's bare `Rejection`, the way a context throws one; that a field named for a command carries a query's name is a gap in the language, and the owner has not ruled on either reading (E-5, D7, Law 5) +- [non-blocking] Extension E-5: the doc says a query authorizes before it discloses and does not say how a refused query is answered; here a parent query's refusal is the same `RejectionData` a command's is, with the name of the entry that refused in the field `entry` and only the codes `unauthenticated` and `forbidden`, and a use case's own rejection is thrown as the kernel's bare `Rejection`, the way a context throws one; the owner confirmed the shared shape, the two codes, the bare use-case rejection and the word entry for the field (E-5, D7, Law 5) - [non-blocking] Extension E-35: the write pause of D9 refuses writers in its scope while a cross-stream history view rebuilds; the doc does not say how a refused writer is answered; this contract reserves the transient code `writePaused` for Package D so the refusal is retryable and stores nothing, which Package D confirms or replaces (E-35, D9, D4) - [non-blocking] OQ2: whether the product needs a record of refused commands anywhere, for security audit or agent proposals, decides whether the generic internal dispatcher below exists; until then it is a conditional design and no command uses it (OQ2, D7) @@ -44,7 +44,7 @@ The closed code list has two halves. The platform codes below are closed here an - Applied and business failure are returned as a `CommandResponse` value whose `kind` is `"applied"` or `"businessFailure"`; both commit, because a business failure is something meaningful that belongs in history (D4) - A rejection is thrown as `new ConvexError(data)` with `data.kind` equal to `"rejection"` and a code from the closed list; the throw prevents the mutation from committing and nothing is stored (D7, D4) -- A parent query's refusal is the same rejection: `new ConvexError(data)` with `data.kind` equal to `"rejection"`, a platform code, and the query's name in `commandType`; a query throws no domain code, stores nothing and writes no receipt, and a client classifies it with the same test as a command's rejection (D7, Law 5) +- A parent query's refusal is the same rejection: `new ConvexError(data)` with `data.kind` equal to `"rejection"`, a platform code, and the query's name in `entry`; a query throws no domain code, stores nothing and writes no receipt, and a client classifies it with the same test as a command's rejection (D7, Law 5) - A technical failure is any throw that is not a `ConvexError` of this contract, such as a plain `Error`, a validator failure inside a component or an unexpected exception; everything rolls back, nothing is stored as a business outcome, and the boundary never converts it into a rejection (D4, Law 6) - A transient refusal is thrown as `new ConvexError(data)` with `data.kind` equal to `"transient"`; the caller may retry the same intent with the same request key, nothing is stored, and a valid retry can succeed (D4, D6, Sc L1-9) - The `data` of every `ConvexError` this contract throws is a value the `v` validators accept, so `errorDataValidator` below describes it exactly and a client reads it as `error.data` after `error instanceof ConvexError` (D7) @@ -56,7 +56,7 @@ The closed code list has two halves. The platform codes below are closed here an - If the response to a rejection is lost and the command runs again, the retry runs against current state and nothing of the first attempt remains; if it now succeeds, the intent ran once (D7, Sc L1-12) - A transient refusal or a rejection never leaves a receipt, so its idempotency key stays unused and the same key can carry a later successful execution (D6, D7) - The code list is closed: the platform codes are the `errorCode` entries below, the domain codes are the `rejections` a command's declaration names, and a code outside both is a technical failure by definition (D4, D12, E-5) -- The outermost catch treats a `ConvexError` as a rejection candidate when its data is an object that is not an array and either has an own `code` property and no own `kind` property, the bare form, or has `kind` equal to `"rejection"`, the wire form; a candidate whose data does not fit the kernel's `rejectionValidator` for the bare form or the rejection branch of `errorDataValidator` for the wire form becomes a plain `Error`; to fit is to validate as Convex validates a value, so an array is not a record; a candidate that fits passes the same three checks: its code is a platform code or one the declaration's `rejections` names, its `commandType`, when it carries one, is the running command's name, and its `details`, when present, measure at most 16,384 bytes with `getConvexSize`; a candidate that fails any of the three is rethrown as a plain `Error`, a technical failure, so no code, command name or details reach the wire that the running command's declaration does not answer for; every other throw passes unchanged (D7, D4, D12, Law 6, E-5) +- The outermost catch treats a `ConvexError` as a rejection candidate when its data is an object that is not an array and either has an own `code` property and no own `kind` property, the bare form, or has `kind` equal to `"rejection"`, the wire form; a candidate whose data does not fit the kernel's `rejectionValidator` for the bare form or the rejection branch of `errorDataValidator` for the wire form becomes a plain `Error`; to fit is to validate as Convex validates a value, so an array is not a record; a candidate that fits passes the same three checks: its code is a platform code or one the declaration's `rejections` names, its `entry`, when it carries one, is the running entry's name, and its `details`, when present, measure at most 16,384 bytes with `getConvexSize`; a candidate that fails any of the three is rethrown as a plain `Error`, a technical failure, so no code, command name or details reach the wire that the running command's declaration does not answer for; every other throw passes unchanged (D7, D4, D12, Law 6, E-5) - The generic internal dispatcher exists only if OQ2 is answered yes; it is one `internalMutation` for every command that needs a refusal record, its nested body is the command's existing internal entry and never a registered body per command, and it returns the rejection as a value because throwing would roll the record back (D7, OQ2, F3, D12) ## Design @@ -68,13 +68,13 @@ The conditional dispatcher is the one place on the parent side where D7's nested - transactionBoundary: the pipeline's one top-level mutation; the conditional dispatcher adds one nested `ctx.runMutation` around the body for the commands that need a refusal record (D7, F3) - convexSurface: no registered function of its own; the error classes and validators are a library the pipeline imports; the dispatcher, if OQ2 says yes, is one `internalMutation` (D7, OQ2) - typeOutcomeMapping: `Outcome` of `spec:kernel.outcome-model` maps as applied → `CommandResponse` kind `"applied"`, businessFailure → `CommandResponse` kind `"businessFailure"`, rejection → thrown `ConvexError`; technical failure is any other throw (D4, D7) -- typeRejectionData: `type RejectionData = Rejection & { kind: "rejection"; code: RejectionCode; commandType: string }` where `Rejection` is the kernel's `{ code: string; message: string; details?: Record }` from `spec:kernel.outcome-model`, never redefined here (D7, E-5) +- typeRejectionData: `type RejectionData = Rejection & { kind: "rejection"; code: RejectionCode; entry: string }` where `Rejection` is the kernel's `{ code: string; message: string; details?: Record }` from `spec:kernel.outcome-model`, never redefined here, and `entry` is the name of the public command or parent query that refused (D7, E-5) - typeTransientData: `type TransientData = { kind: "transient"; code: TransientCode; message: string; retryAfterMs?: number }` (D4, E-5) - typeRejectionCode: `type RejectionCode = PlatformRejectionCode | Domain` where `Domain` is the union of every code the command's declaration names in `rejections`, open as `string` by default (D4, D12, E-5) - typePlatformRejectionCode: `type PlatformRejectionCode = "invalidInput" | "unauthenticated" | "forbidden" | "idempotencyConflict" | "staleVersion" | "entityExists" | "operationTooLarge" | "unsupportedContractVersion"` (E-5) - typeTransientCode: `type TransientCode = "rateLimited" | "capacity" | "writePaused"` (D4, D6, E-35) -- validatorErrorData: `errorDataValidator = v.union(v.object({ kind: v.literal("rejection"), code: v.string(), message: v.string(), commandType: v.string(), details: v.optional(v.record(v.string(), v.any())) }), v.object({ kind: v.literal("transient"), code: v.union(v.literal("rateLimited"), v.literal("capacity"), v.literal("writePaused")), message: v.string(), retryAfterMs: v.optional(v.number()) }))` where the rejection branch is the kernel's `rejectionValidator` plus `kind` and `commandType`, and `code` is `v.string()` at the validator and the closed union at the type, because the domain half is closed per declaration; to fit a validator is to validate as Convex validates a value, so an array is not a record (D7, E-5) -- fnNormalizeThrown: [extension] `normalizeThrown(error: unknown, commandType: string, rejections: readonly string[]): never` rethrows, where `commandType` is the running command's name: a `ConvexError` is a rejection candidate when its `data` is an object that is not an array and either has an own `code` property and no own `kind` property, the bare form, or has `kind` equal to `"rejection"`, the wire form; a candidate whose `data` does not fit its shape, the kernel's `rejectionValidator` for the bare form and the rejection branch of `errorDataValidator` for the wire form, is rethrown as a plain `Error` whose message names `commandType`; to fit is to validate as Convex validates a value, so an array is not a record; every candidate that fits its shape, bare or in the wire shape, passes three checks in this order, and the first that fails rethrows a plain `Error` whose message names `commandType` and what failed: first, `data.code` is a platform code or in `rejections`, and the message names the code; second, for a rejection in the wire shape, `data.commandType` equals `commandType`, and the message names the command name the rejection carried; third, `data.details` is absent or measures at most 16,384 bytes with `getConvexSize`, and the message names the measured byte count in decimal digits with no separator; a bare rejection that passes is rethrown as `new ConvexError({ kind: "rejection", commandType, ...data })`, and a rejection in the wire shape that passes is rethrown as the same error object; a `ConvexError` carrying `kind` equal to `"transient"`, a `ConvexError` with any other data and every other thrown value are rethrown as the same value; it is called once, in the pipeline handler's outermost catch (D7, F14, Law 6, E-5) +- validatorErrorData: `errorDataValidator = v.union(v.object({ kind: v.literal("rejection"), code: v.string(), message: v.string(), entry: v.string(), details: v.optional(v.record(v.string(), v.any())) }), v.object({ kind: v.literal("transient"), code: v.union(v.literal("rateLimited"), v.literal("capacity"), v.literal("writePaused")), message: v.string(), retryAfterMs: v.optional(v.number()) }))` where the rejection branch is the kernel's `rejectionValidator` plus `kind` and `entry`, and `code` is `v.string()` at the validator and the closed union at the type, because the domain half is closed per declaration; to fit a validator is to validate as Convex validates a value, so an array is not a record (D7, E-5) +- fnNormalizeThrown: [extension] `normalizeThrown(error: unknown, entry: string, rejections: readonly string[]): never` rethrows, where `entry` is the running entry's name: a `ConvexError` is a rejection candidate when its `data` is an object that is not an array and either has an own `code` property and no own `kind` property, the bare form, or has `kind` equal to `"rejection"`, the wire form; a candidate whose `data` does not fit its shape, the kernel's `rejectionValidator` for the bare form and the rejection branch of `errorDataValidator` for the wire form, is rethrown as a plain `Error` whose message names `entry`; to fit is to validate as Convex validates a value, so an array is not a record; every candidate that fits its shape, bare or in the wire shape, passes three checks in this order, and the first that fails rethrows a plain `Error` whose message names `entry` and what failed: first, `data.code` is a platform code or in `rejections`, and the message names the code; second, for a rejection in the wire shape, `data.entry` equals `entry`, and the message names the entry the rejection carried; third, `data.details` is absent or measures at most 16,384 bytes with `getConvexSize`, and the message names the measured byte count in decimal digits with no separator; a bare rejection that passes is rethrown as `new ConvexError({ kind: "rejection", entry, ...data })`, and a rejection in the wire shape that passes is rethrown as the same error object; a `ConvexError` carrying `kind` equal to `"transient"`, a `ConvexError` with any other data and every other thrown value are rethrown as the same value; it is called once, in the pipeline handler's outermost catch (D7, F14, Law 6, E-5) - fnReject: [extension] `reject(data: Omit, "kind">): never` throws `new ConvexError({ kind: "rejection", ...data })`, or a plain `Error` when `data.details` measure above 16,384 bytes with `getConvexSize`; `RejectionCode` is the platform half alone, so `data.code` is typed `PlatformRejectionCode` and a call with any other code does not compile; the helper throws the platform's own rejections, and a domain rejection is thrown as the kernel's bare `Rejection`, which the outermost catch checks against the declaration (D7, D12, E-5) - fnRefuseTransient: `refuseTransient(data: Omit): never` throws `new ConvexError({ kind: "transient", ...data })` (D4) - fnClassify: `classifyThrown(error: unknown): { kind: "rejection"; data: RejectionData } | { kind: "transient"; data: TransientData } | { kind: "technical"; error: unknown }` used by tests, the dispatcher and the obligation wrapper, never by the public pipeline, which relays every throw; a `ConvexError` whose `data` does not fit `errorDataValidator`, a bare kernel `Rejection` included, is technical; to fit is to validate as Convex validates a value, as `fnNormalizeThrown` states, so `data` or `details` that are an array are not a record and such an error is technical (D7, Law 6) diff --git a/design/specs/context/context-component.sdp.md b/design/specs/context/context-component.sdp.md index ae17e04..ec9f25c 100644 --- a/design/specs/context/context-component.sdp.md +++ b/design/specs/context/context-component.sdp.md @@ -30,7 +30,7 @@ Layer 1 · Detail: full · Traces: D2, D5, D7, D8, D10, D11, D12, Law 1, Law 2, A bounded context is one Convex component. It owns its current state, its event journal and its stream metadata, and only its own mutations write them. The parent mounts one component per context, authenticates and authorizes, and calls the context through the component API with the tenant, the server-established actor and the operation's identity as arguments. Each such call is an isolated sub-transaction that commits with the caller and rolls back on a throw. The context exposes sanctioned operations that run the persistence adapter over the domain kernel, and authorized queries that return DTOs. It exposes no generic write, no table and no document ID. -The component's boundary is the component API. Below it sit the kernel, the journal library and the tables; above it sit the parent's use cases, which are the only callers, and the parent's queries, which read through the context's queries. Contexts never read each other's tables and never call each other. The cost of the boundary is one component call per parent read or write of context data, whose size Probe 3 measures and whose answer decides whether a trivial context may be plain tables (OQ1). Isolation itself stays. +The component's boundary is the component API. Below it sit the kernel, the journal library and the tables; above it sit the parent's use cases, which are the only callers, and the parent's queries, which read through the context's queries. Contexts never read each other's tables and never call each other. The cost of the boundary is one component call per parent read or write of context data, which Probe 3 and the first experiment measured and which keeps every context a component (OQ1). Isolation itself stays. ## Intent @@ -38,7 +38,7 @@ The component's boundary is the component API. Below it sit the kernel, the jour - problem: A journal any parent code can write lets state change without its events; a central store puts a counter in every write's path; a context that reads `ctx.auth` decides authority twice; and without a boundary a stale reviewed version or a competing command is silently reinterpreted against fresh state (D2, D11, Sc L1-1, Sc L1-10, Sc L1-11) - outcome: Each context is a component that owns its CMS, journal and stream metadata behind sanctioned operations; the parent calls it through the component API with tenant, actor and operation as arguments, and every call commits or rolls back with the caller (D2, D11, F2) - value: Isolation is the platform's defining property; a context's private schema stays private, a cross-context reference is an application identity, and the rule that state never changes without its events is enforced where trusted code cannot break it by mistake (D2) -- risk: Every read of context data from the parent is a component call, which Probe 3 read on a local backend as about 2 ms more than a helper call and whose cost in function-call quota is unknown; if it breaks the read budgets, OQ1 changes where hot reads land, and this Spec flags the dependency without pre-deciding it (F4, Probe 3, OQ1) +- risk: Every read of context data from the parent is a component call, which `spec:facts.f19-nested-calls-draw-on-a-system-operation-budget` reads on the fixture composition as 3.17 ms over a helper call on a quiet local backend, with every cost and speed target of the first experiment held; its cost in function-call quota is a hosted usage reading, and a measured workload that misses an agreed target, or a usage reading above an agreed share, reopens where hot reads land (F4, F19, Probe 3, OQ1) - risk: The standing cost per context is one component with its own tables, one registered mutation per sanctioned operation, up to four queries per stream type, one per definer it calls, one operation query, one maintenance mutation and a dependency on `convex-helpers` for pagination inside the component (D2, Decision method rule 2, E-22, E-24) - assumption: Component calls commit or roll back with the calling mutation, and each is a sub-transaction isolated from other calls (F2, S2) - assumption: The documentation says a component has no `ctx.auth`; the design does not rest on it, because by its own rule no component function reads `ctx.auth` and the parent passes actor and scope (F11, D11) @@ -46,7 +46,7 @@ The component's boundary is the component API. Below it sit the kernel, the jour ### Open questions -- [non-blocking] OQ1: whether a trivial context with no invariants of its own may be plain tables in the parent behind lint rules instead of a component is decided after the experiment measures component call cost; the design assumes a component per context everywhere (OQ1, D2) +- [non-blocking] OQ1: a trivial context with no invariants of its own stays a component; the owner answered OQ1 on the first experiment's local measurement, with the hosted usage reading as the one measurement still to take, and a measured workload that misses an agreed target or a usage reading above an agreed share reopens it (OQ1, D2) - [non-blocking] Probe 3: the cost of one component call in latency and function-call quota decides where hot reads land; the latency is an order of magnitude read on a local backend, the quota needs a hosted deployment to read, and isolation itself stays (Probe 3, F4, D2, D8) - [non-blocking] Extension E-22: the doc says the parent calls the context's sanctioned operations and does not say how they are registered; this Spec registers one Convex mutation per sanctioned operation built by `defineOperation` with args `{ tenantId, actor, operation, input, facts }`, fixes the context ID as a code constant equal to the mount name, sets the per-call bounds in documents and in bytes derived from each stream type's budget, names the query and maintenance surface, and adds `convex-helpers` as the component's one dependency because the built-in `.paginate()` does not work in a component; the alternative of one generic `execute` mutation per context with an untyped command was not taken because D12 wants static exports with one canonical schema; the owner confirms (D10, D12, S4, E-22) @@ -112,7 +112,7 @@ One component per context, one mount per context in the parent, and the parent a - limitReadsPerCall: one stream row plus its parts plus one index range per stream, so a call at the stream bound reads under 300 documents for single-mapping streams and at most 33 per stream, under 8,500, for derived ones, against the 32,000 ceiling, and, by the byte bound above, under 8 MiB of stream documents against the 16 MiB read ceiling; the limits are one budget across the parent and its components, as Probe 4 showed (F13, Probe 4, E-2, E-22) - historyMode: `history: "rebuildable" | "auditOnly"` on the journal; an audit-only context registers no `rebuild` query and its documentation never advertises rebuild from events (D5) - crossContextReference: `type EventRef = { tenantId: string; contextId: string; eventId: string }` is the only shape a context stores about another context's events (D2) -- probeDependency: the shape of this component is what Probe 3 and OQ1 test; a result that breaks the read budgets changes where hot reads land before Layer 3 is designed (F4, Probe 3, OQ1) +- probeDependency: the shape of this component is what Probe 3 and OQ1 tested; a measured workload that misses an agreed target, or a hosted usage reading above an agreed share, changes where hot reads land before Layer 3 is designed (F4, Probe 3, OQ1) ## Example space diff --git a/design/specs/context/journal.sdp.md b/design/specs/context/journal.sdp.md index f4cc74f..a7d0af9 100644 --- a/design/specs/context/journal.sdp.md +++ b/design/specs/context/journal.sdp.md @@ -13,6 +13,7 @@ relations: - spec:laws.law10-replay-never-runs-commands-or-effects - spec:facts.f01-serializable-mutations-under-occ - spec:facts.f13-transactions-have-limits + - spec:facts.f18-commit-timestamps - spec:laws.law02-state-and-events-commit-together - spec:laws.law06-technical-failure-never-a-rejection - spec:laws.law11-tenant-scope-named @@ -24,7 +25,7 @@ relations: Layer 1 · Detail: full · Traces: D2, D3, D5, D18, D19, Law 2, Law 3, Law 6, Law 10, Law 11, F1, F2, F8, F9, F13, S4, Existing systems, Sc L1-10, Sc L2-7, E-2, E-3, E-12, E-25. -The journal is a context's store of its events and the shared library that writes and reads it. An append names the stream version it expects and checks it by an indexed read in the same transaction. Order exists only within a stream; no deployment-wide token exists, and events in different streams relate through the operation and the cause they carry. Domain streams are rebuildable from their events by default, and the tool that keeps rebuild true through a change of meaning is the baseline event, which a migration writes and from which rebuild starts. Old events are never rewritten, and old reducer versions are not kept forever: a migration that changes meaning is registered on its stream type with the function that maps the saved state to the new meaning, and it retires once every stream has its baseline. The baseline is written by one journal helper with two callers: the persistence adapter, on the first command that loads a stream saved under an older meaning, and the parent's driver, which sweeps the streams no command reaches. No decision is therefore taken against a state of the wrong meaning, and nothing pauses writes. +The journal is a context's store of its events and the shared library that writes and reads it. An append names the stream version it expects and checks it by an indexed read in the same transaction. Order within the platform exists only within a stream; the engine's commit timestamp orders transactions and nothing in the journal reads it, and events in different streams relate through the operation and the cause they carry. Domain streams are rebuildable from their events by default, and the tool that keeps rebuild true through a change of meaning is the baseline event, which a migration writes and from which rebuild starts. Old events are never rewritten, and old reducer versions are not kept forever: a migration that changes meaning is registered on its stream type with the function that maps the saved state to the new meaning, and it retires once every stream has its baseline. The baseline is written by one journal helper with two callers: the persistence adapter, on the first command that loads a stream saved under an older meaning, when every step derives the new state from the saved state alone; and the parent's driver, which sweeps every stream. A migration that reads history is written by the sweep alone: until the sweep reaches a stream, a command on it is refused as transient and a read of it is refused the same way, so no command ever replays history, no stream the sweep has passed is held, and nothing pauses writes. The journal's boundary is the library API the persistence adapter and the queries call. The library exports `append`, `load`, `rebuild` and `defineMaintenance`, which builds the baseline writer, and nothing that patches or deletes an event. It is included by every context as code; no context shares a table with another. The baseline writer is a public function of the component, because a component's internal functions are not exposed to the parent, and the parent drives it in bounded batches from a scheduled internal mutation of its own. @@ -35,19 +36,19 @@ The journal's boundary is the library API the persistence adapter and the querie - outcome: Every append is ordered within its stream by an indexed check in the same transaction, events in different streams relate through operation and cause, and a rebuild from the latest baseline gives the saved state while earlier events stay readable (D2, D5, Sc L2-7) - value: Rebuild stays a test rather than a hope, without rewriting history or keeping every reducer, and without any global order (D5, D2) - risk: A change of meaning costs a migration that writes a baseline per affected stream; a live stream pays it on its first command after the deploy, and a cold stream waits for the driver's sweep, during which its rebuild is refused as awaiting its baseline rather than folded under the wrong meaning (D5, E-25) -- risk: A migration that reads history makes the first command on each of its streams read that stream's tail, at most `maxFoldEvents`; an operation that plans many such streams in that window can exceed the read ceiling and fails as a technical failure until the sweep reaches them, so a history-reading migration is the exception (E-25, F13, Law 6) +- risk: A migration that reads history holds each stream of its type until the sweep reaches it: a command or a read on such a stream is refused as transient, with the code `writePaused`, and the caller retries after the sweep; the cost is the sweep's duration on the streams it has not reached, no stream type is held in whole, and no command reads a tail, so a history-reading migration is the exception and D3 holds (E-25, D3, F13, Law 6) - risk: A rebuild reads the events after the latest baseline in one transaction; a stream whose tail exceeds the fold bound is refused rather than folded and needs a baseline or the snapshot tier trigger, and the 1 second query timeout binds before the read ceiling, so the bound is provisional until the first experiment measures a fold (F13, D18, E-25) -- risk: A baseline is an event at the next stream version, so a migration advances every migrated stream by one; a caller holding a reviewed version across the sweep is answered `staleVersion` on its next command and re-reviews from the `current` the details carry, every read-model row whose source the sweep touched has `sourceVersions` one behind its stream until the next command or verify pass rewrites it, and a client comparing the versions its last command returned with a subscription sees a version it never wrote; a meaning change is therefore rolled out in a maintenance window or under the tenant's write pause, and the driver reads the gate before every batch (D2, D5, Sc L1-10, E-25, E-8) +- risk: A baseline is an event at the next stream version, so a migration advances every migrated stream by one; a caller holding a reviewed version across the sweep is answered `staleVersion` on its next command and re-reviews from the `current` the details carry, every read-model row whose source the sweep touched has `sourceVersions` one behind its stream until the next command or verify pass rewrites it, and a client comparing the versions its last command returned with a subscription sees a version it never wrote; a state-only meaning change therefore pauses nothing and its rollout states the version bump as its cost, and the driver reads the gate before every batch (D2, D5, Sc L1-10, E-25, E-8) - assumption: Mutations are serializable under optimistic concurrency, so the indexed read that checks the version puts the stream in the read set and the engine serializes concurrent appends (F1) ### Open questions -- [non-blocking] Extension E-25: the doc says a migration writes a baseline event holding the migrated state and that rebuild starts from the latest baseline, and it does not say who computes that state, who drives the migration, or what a live command does on a stream the migration has not reached; this Spec fixes the representation, an event of the reserved type `baseline` whose payload holds the state and its schema version and is bounded by the stream's `budgetBytes` instead of the payload bound, a `baselineVersion` on the stream row, a `BaselineMigration` registered on the stream type whose `migrate` function computes the migrated state from the saved state and optionally the events since the last baseline, the rule that `stateSchemaVersion` counts meaning only and lives on the registration, one helper `migrateToSchemaVersion` that writes the baseline chain, called by the adapter on load for a live stream and by the `writeBaseline` maintenance mutation for the sweep, the code-older-than-data error, the `writeBaseline` mutation built by `defineMaintenance` and declared with the component's public `mutation` builder, the parent-side scheduled chain that drives it by cursor, the batch bound derived from `budgetBytes`, the upcasting rule for old event schema versions, the fold bound of 500 events with the refusals `rebuildStream` returns above it and for a stream awaiting its baseline, the retirement rule for a migration, and the payload byte check in `append`; the option of refusing commands on unmigrated streams with a transient code until the sweep arrives was not taken because it turns a deploy into a partial write pause the doc reserves for history-dependent views; a consequence the owner should weigh is that every baseline bumps its stream's version, so a reviewed version held across the sweep is answered `staleVersion` and re-reviewed rather than silently accepted, because Sc L1-10 forbids reinterpreting a reviewed version against fresh state and the baseline is a real event of the stream, while a live stream migrated on load never trips it because its expected-version check runs before the chain; and the driver reads the write pause's gate before every batch and reschedules itself when refused, so a sweep never changes a source under a paused rebuild's cut; the owner confirms (D5, D9, S4, Sc L1-10, E-2, E-8, E-12, E-25) +- [non-blocking] Extension E-25: the doc says a migration writes a baseline event holding the migrated state and that rebuild starts from the latest baseline, and it does not say who computes that state, who drives the migration, or what a live command does on a stream the migration has not reached; this Spec fixes the representation, an event of the reserved type `baseline` whose payload holds the state and its schema version and is bounded by the stream's `budgetBytes` instead of the payload bound, a `baselineVersion` on the stream row, a `BaselineMigration` registered on the stream type whose `migrate` function computes the migrated state from the saved state, or, for a step that declares `readsHistory`, from the saved state and the events since the last baseline in the sweep only, the rule that `stateSchemaVersion` counts meaning only and lives on the registration, one helper `migrateToSchemaVersion` that writes the baseline chain, called by the adapter on load for a chain that reads no history and by the `writeBaseline` maintenance mutation for every chain, the code-older-than-data error, the `writeBaseline` mutation built by `defineMaintenance` and declared with the component's public `mutation` builder, the parent-side scheduled chain that drives it by cursor, the batch bound derived from `budgetBytes`, the upcasting rule for old event schema versions, the fold bound of 500 events with the refusals `rebuildStream` returns above it and for a stream awaiting its baseline, the retirement rule for a migration, and the payload byte check in `append`; the option of refusing commands on unmigrated streams with a transient code until the sweep arrives was not taken for a state-only migration, because it turns a deploy into a partial write pause the doc reserves for history-dependent views, and is taken for a history-reading migration, the exception, where it holds only the streams the sweep has not reached and keeps D3 whole on the command path, which the owner confirmed; a consequence the owner should weigh is that every baseline bumps its stream's version, so a reviewed version held across the sweep is answered `staleVersion` and re-reviewed rather than silently accepted, because Sc L1-10 forbids reinterpreting a reviewed version against fresh state and the baseline is a real event of the stream, while a live stream migrated on load never trips it because its expected-version check runs before the chain; and the driver reads the write pause's gate before every batch and reschedules itself when refused, so a sweep never changes a source under a paused rebuild's cut; the owner confirms (D5, D9, S4, Sc L1-10, E-2, E-8, E-12, E-25) ## Behavior - rule: An append names the stream version it expects; the check is an indexed read in the same transaction, and an index alone is not a unique constraint (D2, F1) -- rule: Order exists only within a stream; no deployment-wide order token exists (D2) +- rule: Order exists only within a stream; the journal keeps no deployment-wide order token, and the engine's commit timestamp, which orders transactions, is read by nothing in it (D2, F18) - rule: Timestamps, `_creationTime` and UUID order never prove that a consumer has seen every committed event (D2) - rule: Events in different streams relate through `operationId` and `causedBy`, and a receipt may list the stream versions one operation touched (D2) - rule: The starting indexes are stream replay, event identity and stream enumeration; others are added when a real query needs them (D2) @@ -63,16 +64,16 @@ The journal's boundary is the library API the persistence adapter and the querie - rule: [extension] The journal library exports no function that patches or deletes an event; the events table is append-only by construction of its only writer (E-25, D5) - rule: [extension] A baseline is an event of the reserved type `baseline` at the next stream version, whose payload holds the migrated state and its schema version; `evolve` never sees it, because rebuild takes its payload as the start (E-25, D5) - rule: [extension] A payload whose `eventSchemaVersion` is older than the current one is upcast in `evolve` until a baseline retires it; the stored event is never touched (E-25, D5) -- rule: [extension] A command that loads a stream whose saved `stateSchemaVersion` is behind the registration's runs the registered baseline chain on that stream first, inside its own sub-transaction and before `decide`, so no decision is taken against a state of the wrong meaning; the driver's sweep migrates the streams no command reaches, and nothing pauses writes (E-25, D3, D5, Law 2) +- rule: [extension] A command that loads a stream whose saved `stateSchemaVersion` is behind the registration's runs the registered baseline chain on that stream first, inside its own sub-transaction and before `decide`, when every step derives its state from the saved state alone, and is refused as transient until the sweep has migrated the stream when a step reads history; so no decision is taken against a state of the wrong meaning and no command replays history; the driver's sweep migrates every stream no command reaches, and nothing pauses writes (E-25, D3, D5, Law 2) - rule: [extension] A stream row whose `stateSchemaVersion` is ahead of the registration's, or a chain with a gap between the row's version and the registration's, is a plain error and a technical failure, never a rejection, because the code is older than the data or a migration was not registered (E-25, Law 6) - rule: [extension] `stateSchemaVersion` counts meaning: it is declared once on the stream type's registration, every bump of it is one registered `BaselineMigration`, and a change of representation rewrites rows through the mapping and moves nothing (E-25, D5) - rule: [extension] A `BaselineMigration` leaves its registry only after the sweep has reported no stream below its `toSchemaVersion` in every tenant; until then a cold stream may still need it, and removing it earlier is the gap the rule above names (E-25, D5) - flow: The adapter loads the stream row by the identity index and takes its version as the expected version; when the command names a reviewed version, the adapter compares it with the row's version and rejects a mismatch as `staleVersion` before `decide` runs, which is step 3 of `spec:context.persistence-adapter` (D2, Sc L1-10) -- flow: When the loaded row's `stateSchemaVersion` is behind the registration's, the adapter calls `migrateToSchemaVersion`, which appends one baseline event per registered step at the next versions, carrying the command's operation and actor and `causedBy` of kind `migration`, and hands back the migrated state at the advanced version; the command's events follow the baselines and the row is saved once, which is step 3a of `spec:context.persistence-adapter` (D5, D3, Law 2, E-25) +- flow: When the loaded row's `stateSchemaVersion` is behind the registration's and no step of the chain reads history, the adapter calls `migrateToSchemaVersion`, which appends one baseline event per registered step at the next versions, carrying the command's operation and actor and `causedBy` of kind `migration`, and hands back the migrated state at the advanced version; the command's events follow the baselines and the row is saved once, which is step 3a of `spec:context.persistence-adapter` (D5, D3, Law 2, E-25) - flow: Before writing, `append` reads the events index from the expected version up, which puts the stream's range in the transaction's read set so the engine serializes concurrent appends; an event above the expected version, or no event at a nonzero expected version, means the stream row and the events index disagree, which is a defect that throws a plain error, never a rejection, because the caller-named case was already answered from the row (D2, F1, Law 6) - flow: The adapter inserts the new events at consecutive versions through `append` and then saves the stream row once with the last version, at its step 9, in the same transaction (D2, Law 2) - flow: A rebuild reads the baseline event at the row's `baselineVersion` when there is one, folds `evolve` over the events with a greater version from the baseline's state or from `initial()`, and compares state and version with the saved row (D3, D5, Sc L2-7) -- flow: A migration that changes meaning is registered on its stream type as a `BaselineMigration` with a name, the state schema versions it goes from and to, and a `migrate` function, and the registration's `stateSchemaVersion` is raised to its target in the same deploy; from that deploy on, live streams migrate on their first command and the parent's driver calls `maintenance.writeBaseline` through `ctx.runMutation` once per bounded batch and schedules itself with the returned cursor until the enumeration is done (D5, D19, F2, F8, E-25) +- flow: A migration that changes meaning is registered on its stream type as a `BaselineMigration` with a name, the state schema versions it goes from and to, and a `migrate` function, and the registration's `stateSchemaVersion` is raised to its target in the same deploy; from that deploy on, a state-only migration is written for live streams by their first command and for the rest by the parent's driver, a history-reading migration by the driver alone, and the driver calls `maintenance.writeBaseline` through `ctx.runMutation` once per bounded batch and schedules itself with the returned cursor until the enumeration is done (D5, D19, F2, F8, E-25) - flow: Each `writeBaseline` batch enumerates the tenant's streams of that type after the cursor, skips every stream already at or past the target version, which a live command or an earlier batch brought there by writing its baseline, and for every other stream calls the same `migrateToSchemaVersion` helper, which computes the migrated state with `migrate` from the loaded state and, when the migration asks for it, the events since the last baseline, and inserts one baseline event per step at the next versions; the batch then saves the row, all in one sub-transaction, and never rewrites an earlier event (D5, D19, Law 2, E-25) ## Design @@ -83,7 +84,7 @@ The library is generic over the component's data model and is instantiated once - fnCreateJournal: `createJournal(config: { contextId: string; history: "rebuildable" | "auditOnly" }): Journal` returns the configuration frozen (D2, D5, E-25) - typeJournal: `type Journal = { readonly contextId: string; readonly history: "rebuildable" | "auditOnly" }` (D2, D5) - typeStreamRegistration: `type StreamRegistration = { decider: Decider; mapping: StateDocumentMapping; stateSchemaVersion: number; eventValidators: Record>; dto: Validator; toDto: (state: S, meta: StreamMeta) => Value; baselineMigrations: readonly BaselineMigration[] }` declared once per stream type, where `stateSchemaVersion` is the version the current `evolve` produces, `mapping.budgetBytes` is the stream document budget every byte-derived bound reads, `dto` is the validator of what `toDto` returns, `toDto` carries `meta`'s stream version on the DTO as `version: StreamVersion` as `spec:context.queries` pins, and `baselineMigrations` is the static registry of the migrations that may write a baseline for this stream type (D2, D3, D5, D8, E-2, E-24, E-25) -- typeBaselineMigration: `type BaselineMigration = { name: string; fromSchemaVersion: number; toSchemaVersion: number; readsHistory: boolean; migrate: (saved: unknown, history: readonly EventEnvelope[]) => S }` where `saved` is the stream's state as `load` assembles it under the old meaning and `history` is the events after the stream's latest baseline when `readsHistory` is true, otherwise an empty array; the registered migrations form one chain, each step's `fromSchemaVersion` equal to the previous step's `toSchemaVersion` and the last step's `toSchemaVersion` equal to the registration's `stateSchemaVersion` (D5, E-25) +- typeBaselineMigration: `type BaselineMigration = { name: string; fromSchemaVersion: number; toSchemaVersion: number; readsHistory: boolean; migrate: (saved: unknown, history: readonly EventEnvelope[]) => S }` where `saved` is the stream's state as `load` assembles it under the old meaning and `history` is the events after the stream's latest baseline when `readsHistory` is true, otherwise an empty array; the registered migrations form one chain, each step's `fromSchemaVersion` equal to the previous step's `toSchemaVersion` and the last step's `toSchemaVersion` equal to the registration's `stateSchemaVersion`; a step with `readsHistory` true is run by `writeBaseline` only (D5, E-25) - migratedStateSource: the migrated state is `migrate(saved, history)`; the saved state is the input because it is the old reducer's fold, kept current by every command, so the migration needs no old reducer and reads events only when it declares `readsHistory` to derive something the old state never held (D3, D5, E-25) - typeStreamMeta: `type StreamMeta = { tenantId: string; contextId: string; streamType: string; streamId: string; streamVersion: number; stateSchemaVersion: number; deletedAt?: number; baselineVersion?: number }` where `stateSchemaVersion` is the meaning version the row was saved under, which the adapter compares with the registration's on load (D2, E-25) - typeLoadedStream: `type LoadedStream = { state: S; version: number; exists: boolean; meta: StreamMeta; rowId?: GenericId<"streams">; parts?: Record }` where `rowId` is the stream row's document ID, set when the row exists, by which the adapter's step 9 replaces the row (D3, E-2, E-23) @@ -97,8 +98,8 @@ The library is generic over the component's data model and is instantiated once - rebuildReads: the stream row first, by `by_identity`; when its `stateSchemaVersion` is behind the registration's the function returns the refusal `awaitingBaseline` without reading an event, because the current `evolve` cannot fold the old meaning and the stream's baseline is still to come from a command or the sweep; when `streamVersion - (baselineVersion ?? 0)` exceeds `maxFoldEvents` it returns `tailTooLong` without reading an event; when it exceeds `maxEvents` but not `maxFoldEvents` it returns `budgetExhausted` without reading an event, which is a resume point for a batching caller and never a finding about the stream; otherwise `by_stream` with `.eq` on the three identity fields and `.gte("streamVersion", baselineVersion ?? 1)` in ascending order, collected in one transaction, the baseline event first when there is one (D5, F13, E-25, E-42) - limitFoldEvents: [extension] `maxFoldEvents` is 500 events since the latest baseline, provisional; at `limitEventBytes` of `spec:context.event-envelope`, 20,480 bytes and the 61 bytes of a stored document's system fields, a fold reads at most 10,270,500 bytes of events, under 9.8 MiB, and at most 11,323,294 bytes, under 10.9 MiB, with the stored stream row of at most `budgetBytes + 61` bytes for a single mapping, where a derived mapping adds 61 bytes for each of its documents, and the stored baseline event of at most `budgetBytes + 4,096 + 61` bytes, at the 512 KiB cap on `budgetBytes`, which leaves more than 5 MiB of headroom under the 16 MiB ceiling, and the 1 second query timeout for folding and deep-comparing that much binds first, so the first experiment and the L2-7 example measure it; the same bound applies to the tail a history-reading migration step reads, and a longer tail is a plain error there (F13, E-2, E-12, E-25) - baselineEvent: `eventType: "baseline"`, `eventSchemaVersion: step.toSchemaVersion`, `payload: { state, stateSchemaVersion: step.toSchemaVersion }` where `state` is what the step's `migrate` returned, `causedBy: { kind: "migration", migrationName: step.name }`, `operationId` and actor those of the driver's run or of the command that migrated the stream on load, at the stream's next version; `evolve` never sees it (D5, E-25, E-26) -- fnMigrateToSchemaVersion: `migrateToSchemaVersion(ctx: MutationCtx, journal: Journal, registration: StreamRegistration, loaded: LoadedStream, envelope: EnvelopeInput, toSchemaVersion: number): Promise>` is the one writer of baseline events; it throws a plain error when `loaded.meta.stateSchemaVersion` is above `toSchemaVersion` or when the registered chain from that version to `toSchemaVersion` has a gap, and otherwise runs the chain step by step: read the tail by `by_stream` above `baselineVersion ?? 0` when the step reads history, a tail above `maxFoldEvents` being a plain error, compute `step.migrate(saved, history)`, `append` one baseline event at `loaded.version + 1`, and continue from the migrated state; it returns the loaded stream at the advanced version with `baselineVersion` and `stateSchemaVersion` updated on `meta`, and saves no row, which its caller does once (D5, D3, Law 2, Law 6, E-25) -- migrateToSchemaVersionCallers: the persistence adapter calls it at step 3a with `registration.stateSchemaVersion` for a stream whose row is behind, before `decide` and after the expected-version check, so the caller's reviewed version is compared with what the caller could have seen; `writeBaseline` calls it with the named migration's `toSchemaVersion` for every stream of its batch that is behind; both paths write the same events and the same row fields, so state and history agree whichever path a stream took (D3, D5, Sc L1-10, E-25) +- fnMigrateToSchemaVersion: `migrateToSchemaVersion(ctx: MutationCtx, journal: Journal, registration: StreamRegistration, loaded: LoadedStream, envelope: EnvelopeInput, toSchemaVersion: number): Promise>` is the one writer of baseline events; it throws a plain error when `loaded.meta.stateSchemaVersion` is above `toSchemaVersion` or when the registered chain from that version to `toSchemaVersion` has a gap, and otherwise runs the chain step by step: read the tail by `by_stream` above `baselineVersion ?? 0` when the step reads history, which only `writeBaseline` asks for, a tail above `maxFoldEvents` being a plain error, compute `step.migrate(saved, history)`, `append` one baseline event at `loaded.version + 1`, and continue from the migrated state; it returns the loaded stream at the advanced version with `baselineVersion` and `stateSchemaVersion` updated on `meta`, and saves no row, which its caller does once (D5, D3, Law 2, Law 6, E-25) +- migrateToSchemaVersionCallers: the persistence adapter calls it at step 3a with `registration.stateSchemaVersion` for a stream whose row is behind and whose chain reads no history, before `decide` and after the expected-version check, so the caller's reviewed version is compared with what the caller could have seen; `writeBaseline` calls it with the named migration's `toSchemaVersion` for every stream of its batch that is behind; both paths write the same events and the same row fields, so state and history agree whichever path a stream took (D3, D5, Sc L1-10, E-25) - baselineRow: the caller of `migrateToSchemaVersion` saves the stream row with `baselineVersion` and `streamVersion` from the returned `meta`, `stateSchemaVersion: toSchemaVersion`, `lastOperationId` and the migrated state written through the registration's `mapping`, so a derived mapping writes its parts too, in the same transaction as the baseline inserts; the adapter saves it at step 9 after the command's own events, `writeBaseline` right after the chain (D5, E-2, E-25) - fnDefineMaintenance: `defineMaintenance(journal: Journal, registrations: readonly StreamRegistration[]): { writeBaseline: RegisteredMutation<"public", WriteBaselineArgs, WriteBaselineResult> }`, whose args and result are the validators pinned on `fnWriteBaseline`, resolves the stream type and the migration name against the registrations' `baselineMigrations` at call time, and the component exports its result as `export const { writeBaseline } = defineMaintenance(journal, [orderStream])` in its `maintenance` module (D5, D12, E-25) - fnWriteBaseline: `export const writeBaseline = mutation({ args: { tenantId: v.string(), streamType: v.string(), migrationName: v.string(), actor: actorValidator, operationId: v.string(), correlationId: v.optional(v.string()), afterStreamId: v.optional(v.string()), maxStreams: v.optional(v.number()) }, returns: v.object({ written: v.number(), skipped: v.number(), eventsRead: v.number(), nextAfterStreamId: v.union(v.string(), v.null()) }), handler })` is what `defineMaintenance` builds, with the component's own `mutation` builder (D5, D19, S4, E-25) @@ -110,7 +111,7 @@ The library is generic over the component's data model and is instantiated once - baselineDriver: the driver is a parent-side scheduled internal mutation chain with the same batch contract as the parent's read-model backfill; each batch first reads the write pause's gate through `gateAllows(ctx, ["all", "tenant:" + tenantId, "source:" + contextId + ":" + streamType])` of `spec:application.write-pause`, because a baseline is a write to a source stream, and when the answer is not allowed it reschedules itself after `limitDriverDeferral` with the same cursor and writes nothing; otherwise it makes one call of `ctx.runMutation(components.orders.maintenance.writeBaseline, { tenantId, streamType, migrationName, actor, operationId, afterStreamId })` and `ctx.scheduler.runAfter(0, internal.maintenance.migrateBaselines, { ..., afterStreamId: result.nextAfterStreamId })` until the cursor is `null`; scheduling from a mutation commits with it, a scheduled mutation retries internal errors and runs once, and a developer error ends the chain for the operator to restart, which is safe by idempotence; the driver is a sweep of the streams no command has reached since the deploy, and it races nothing, because a live command that reaches a stream first writes the same baseline and the sweep then skips that stream by its stamp (F2, F8, F9, D9, D19, E-8, E-25) - limitDriverDeferral: 60 seconds between gate reads while a batch is refused, so a closed tenant, `all` or source scope delays the sweep and never ends it, and the outstanding deferred batch is one per chain (E-25, E-8, F13) - sweepVersionBump: the adapter's step 3 compares a reviewed version with the row as loaded and before any baseline of step 3a, so a command that reaches a stream before the sweep never sees `staleVersion` from the migration; a command that reaches it after the sweep does, with `details.current` one higher than it reviewed, and re-reviews, because treating a reviewed version equal to `baselineVersion - 1` as current would reinterpret a reviewed version against a stream that changed, which Sc L1-10 forbids even when the change is a baseline (Sc L1-10, D2, D5, E-25) -- migrationWindow: between the deploy that raises a registration's `stateSchemaVersion` and the end of the sweep, a live stream is migrated by its first command, a cold stream waits for the sweep, the `rebuild` query answers `awaitingBaseline` for a stream not yet migrated instead of folding it under the wrong meaning, and no writer is paused; the window is the sweep's duration, and a migration that reads history is the one case where the first command on a stream costs a tail read (D5, D9, E-25) +- migrationWindow: between the deploy that raises a registration's `stateSchemaVersion` and the end of the sweep, for a state-only migration a live stream is migrated by its first command and a cold stream waits for the sweep; for a history-reading migration every stream waits for the sweep, and a command or a read on one it has not reached is refused as transient; the `rebuild` query answers `awaitingBaseline` for a stream not yet migrated instead of folding it under the wrong meaning, and no writer is paused; the window is the sweep's duration, and no command reads a tail (D5, D9, E-25) - codeOlderThanData: a stream row whose `stateSchemaVersion` is above the registration's is data saved by newer code, which happens after a rollback of a deploy whose sweep or live commands already wrote baselines; `load` and `writeBaseline` throw a plain error for it, the command is a technical failure under Law 6, and the remedy is to redeploy the newer code, never to run the old `evolve` against the newer state (E-25, Law 6, D5) - driverTenancy: the driver runs one tenant and one stream type per chain, because the enumeration index leads with `tenantId` and the parent, not the component, knows the tenants; a meaning change is rolled out by one chain per tenant (Law 11, E-25) - driverNotMigrationsComponent: `@convex-dev/migrations` iterates a table of the deployment that defines the migration and cannot page a component's `streams` table from the parent, so the baseline migration uses the scheduled chain and not that component (S4, E-25) diff --git a/design/specs/context/persistence-adapter.sdp.md b/design/specs/context/persistence-adapter.sdp.md index 9168d0c..6d636ad 100644 --- a/design/specs/context/persistence-adapter.sdp.md +++ b/design/specs/context/persistence-adapter.sdp.md @@ -81,7 +81,7 @@ Everything below runs in the component's mutation context. `MutationCtx` is the - step1: [extension] bound check, in this order and before any read; the runner throws a plain error when a planned command's registration is not among `declaration.streams`, which is a defect, then rejects with `operationTooLarge` when `plan(input).length` exceeds `min(declaration.maxStreams, 256)`, then when the sum of `registration.mapping.budgetBytes` over the planned commands exceeds 8 MiB, then rejects with `invalidInput` and `details` `{ field: "streamId", length, limit: 256 }` for the first planned command, in plan order, whose `streamId` is longer than 256 bytes of UTF-8, `length` being its byte count, and then throws a plain error when the plan holds no command, a defect, because a committed outcome carries at least one event (D10, F13, E-2, E-21, E-22, E-23) - step2: load; `load(ctx, journal, registration, tenantId, target.streamId)` reads the stream row by `by_identity` with its `stateSchemaVersion` and `baselineVersion` on `meta`, assembles a derived state from its parts, or returns `initial()` at version 0 (D3, E-1, E-2, E-25) - step3: expected version; `expected = target.expectedVersion ?? loaded.version`; when `target.expectedVersion === 0` and `loaded.exists`, throw `new ConvexError({ code: "entityExists", message, details: { existing: target.streamId, current: loaded.version } })`; otherwise when `target.expectedVersion` is set and differs from `loaded.version`, throw `new ConvexError({ code: "staleVersion", message, details: { expected: target.expectedVersion, current: loaded.version } })`; both before `decide` runs and both against the version as loaded, before any baseline of step 3a, so a reviewed version is compared with what the caller could have seen (Sc L1-10, Sc L1-4, E-21, E-25) -- step3a: migrate on load; when `loaded.exists` and `loaded.meta.stateSchemaVersion` is below `registration.stateSchemaVersion`, `loaded = await migrateToSchemaVersion(ctx, journal, registration, loaded, envelopeInput, registration.stateSchemaVersion)` of `spec:context.journal`, which appends one baseline event per registered step at `loaded.version + 1` onward with `causedBy: { kind: "migration", migrationName }`, the command's `operationId` and actor, and returns the migrated state at the advanced version with `baselineVersion` and `stateSchemaVersion` updated on `meta`; `expected` advances by the number of baselines so the command's events follow them; when `loaded.meta.stateSchemaVersion` is above `registration.stateSchemaVersion`, or the chain has a gap, the helper throws a plain error and the call is a technical failure (E-25, D3, D5, Law 2, Law 6) +- step3a: migrate on load; when `loaded.exists` and `loaded.meta.stateSchemaVersion` is below `registration.stateSchemaVersion` and a step of the registered chain from that version reads history, the call throws the transient refusal `writePaused` of `spec:command.outcome-boundary`, retryable, until the sweep has migrated the stream; otherwise, when it is below, `loaded = await migrateToSchemaVersion(ctx, journal, registration, loaded, envelopeInput, registration.stateSchemaVersion)` of `spec:context.journal`, which appends one baseline event per registered step at `loaded.version + 1` onward with `causedBy: { kind: "migration", migrationName }`, the command's `operationId` and actor, and returns the migrated state at the advanced version with `baselineVersion` and `stateSchemaVersion` updated on `meta`; `expected` advances by the number of baselines so the command's events follow them; when `loaded.meta.stateSchemaVersion` is above `registration.stateSchemaVersion`, or the chain has a gap, the helper throws a plain error and the call is a technical failure (E-25, D3, D5, Law 2, Law 6) - step4: decide; `registration.decider.decide(loaded.state, target.command, { now: request.now, actor: request.actor, facts: request.facts })` against the state of the current meaning (D3, E-25) - step5: rejection; when the result's `kind` is `rejection`, throw `new ConvexError(result.rejection)` and write nothing (D4, D7, Sc L1-1) - step6: guards; when `result.events.length === 0` throw a plain error; validate each event's payload with `registration.eventValidators[event.eventType]` through `validate` of `convex-helpers/validators`, because `convex/values` exports no run-time validator, throwing a plain error when it answers false or when the event type has no own validator (E-23, D2) @@ -105,7 +105,7 @@ Everything below runs in the component's mutation context. `MutationCtx` is the - errorCodeContextOwned: every other rejection code comes from `decide` and is documented by the context (D4) - limitStreamsPerCall: `min(declaration.maxStreams, 256)` streams and, in bytes, the sum of the planned streams' `budgetBytes`, each counting its row and parts together, at most 8 MiB, half the read ceiling, so a call at the default 256 KiB budget admits 32 streams, a derived stream type at the 512 KiB cap admits 16, and a stock stream declared at 4 KiB admits the full 256 (D10, F13, E-2, E-22) - limitDocumentsWrittenPerCall: 800 documents and 8 MiB, checked by the runner as `sum(appended) + streams + parts written` against 800 and as the sum of each saved stream row's size as step 9 measures it plus every appended event's size measured with `getConvexSize`, baselines of step 3a included, against 8 MiB, before returning; admission at step 1 stays by `budgetBytes`, so a plan at the admitted bound commits when the rows and events it writes measure under 8 MiB; exceeding either throws a plain error so the call is a technical failure and the bound is lowered, never silently split (F13, D19, E-12, E-22, E-25) -- migrationOnLoadCost: a stream migrated at step 3a costs its command one baseline event per step, at most `budgetBytes` each, counted against the write bound above, and when a step reads history a tail of at most `maxFoldEvents`; an operation that plans many streams of a stream type under a history-reading migration can therefore exceed the read ceiling on its first call after the deploy, which fails as a technical failure until the sweep has reached those streams, so a history-reading migration is the exception and the window is the sweep's duration (E-25, F13, Law 6) +- migrationOnLoadCost: a stream migrated at step 3a costs its command one baseline event per step, at most `budgetBytes` each, counted against the write bound above, and never a tail read, because a history-reading step refuses the command as transient until the sweep has reached the stream; the window is the sweep's duration (E-25, F13, Law 6) - limitTimeout: the whole parent mutation has 1 second; the runner records nothing about time, and the first experiment measures the order sizes that fit (F13, First experiment, Sc L2-3) - noSchedulingNoNesting: the adapter schedules nothing and calls no `ctx.runMutation`; helpers are plain functions inside the sub-transaction (S10, D7) diff --git a/design/specs/context/queries.sdp.md b/design/specs/context/queries.sdp.md index 81fa2fd..da85f0d 100644 --- a/design/specs/context/queries.sdp.md +++ b/design/specs/context/queries.sdp.md @@ -35,7 +35,7 @@ A context exposes queries that return DTOs. The parent authenticates and authori - [non-blocking] Probe 5 showed on the pinned backend with `convex-helpers` 0.1.124 that the parent query stays reactive, that pages pinned by their end cursors stay contiguous, that a capped page crosses the boundary as `SplitRequired` with its `splitCursor`, and that the `usePaginatedQuery` hook of `convex-helpers/react` then shows 40 of 45 rows; this design uses no hook across the boundary, and the examples are rerun when either pin moves (Probe 5, F15, S4, D8, E-24) - [non-blocking] Extension E-24: the doc says component queries are authorized and return DTOs; this Spec fixes one definer per query, the one `tenantId` argument with no separate scope, the `toDto` mapping with the stream version on every DTO, pagination through `paginator`, the item cap with a row cap of twice the item cap and a byte cap passed on every run, the relay of `pageStatus` and `splitCursor` unchanged, paging by an explicit cursor pair with no client hook, the parent query's one `authorizeQuery` call and its refusal as a rejection, the `maxEvents` argument of the `rebuild` query and its `budgetExhausted` refusal for a batching caller, and the fold refusal of the `rebuild` query; this is a provisional reading the owner has not ruled (D8, S4, E-24, E-42) -- [non-blocking] A parent query's refusal carries the query's name in the field `commandType` of the rejection data, a field whose name says command; the wire shape stays `RejectionData` of `spec:command.outcome-boundary` unchanged, and whether the field is renamed or the language gains a word for both is the owner's ruling, which has not been made (D7, E-24) +- [non-blocking] A parent query's refusal carries the name of the entry that refused in the field `entry` of the rejection data; the wire shape stays `RejectionData` of `spec:command.outcome-boundary` unchanged, and the owner confirmed the word (D7, E-24) ## Contract @@ -58,7 +58,7 @@ A context exposes queries that return DTOs. The parent authenticates and authori - [extension] A list is ordered by `streamId` within a stream type; a context that needs another order adds an index and a query for it (E-24, D2) - [extension] Every DTO a registration's `toDto` returns carries the stream's `StreamVersion` as `version`, so the parent, the projection and the backfill read one shape on the live path and on the enumeration (E-24, D8, D9) - [extension] A `rebuild` query refuses, with a structured result rather than a read-limit error, a stream whose tail since its latest baseline exceeds the fold bound, and likewise a stream whose saved `stateSchemaVersion` is behind the registration's and so still awaits its baseline; the caller counts either refusal as a journal finding (E-24, E-25, F13) -- [extension] The parent query that relays a context query takes `tenantId` in its own `args`, establishes and authorizes its actor through `authorizeQuery` of `spec:command.actor-and-scope` before the component call, never reads `ctx.auth` itself and never infers the tenant from the identity; a refusal is the rejection of `spec:command.outcome-boundary` with the code `unauthenticated` or `forbidden` and the query's name in `commandType`, and it writes nothing, no receipt included (E-24, Law 11, Law 5, D11, D7) +- [extension] The parent query that relays a context query takes `tenantId` in its own `args`, establishes and authorizes its actor through `authorizeQuery` of `spec:command.actor-and-scope` before the component call, never reads `ctx.auth` itself and never infers the tenant from the identity; a refusal is the rejection of `spec:command.outcome-boundary` with the code `unauthenticated` or `forbidden` and the query's name in `entry`, and it writes nothing, no receipt included (E-24, Law 11, Law 5, D11, D7) - [extension] A cursor the pagination call cannot read is a technical failure, the plain error the call throws, and never a rejection (E-24, D4) ## Design @@ -69,15 +69,15 @@ Each query is built by its own definer from the journal and a stream registratio - fnDefineList: `defineList(journal: Journal, registration: StreamRegistration): RegisteredQuery<"public", ListArgs, Promise>>` registering `{ args: { tenantId: v.string(), paginationOpts: paginationOptsValidator, includeDeleted: v.optional(v.boolean()) }, returns: paginationResultValidator(registration.dto), handler }` (D8, E-24) - typeQueryArgs: `type GetArgs = { tenantId: string; streamId: string }` and `type ListArgs = { tenantId: string; paginationOpts: PaginationOptions; includeDeleted?: boolean }` where `PaginationOptions` is the type of `convex/server` (E-24, Law 11) - fnDefineOthers: `defineHistory(journal, registration)`, `defineRebuild(journal, registration)` and `defineByOperation(journal)` register the `history`, `rebuild` and `byOperation` queries below in the same way, one definer each (D5, D19, E-24) -- fnGet: the handler of `defineGet` reads the stream through the journal's `load`, which reads the row by `by_identity` and assembles a derived state from its parts, returns `null` when the stream does not exist or its row carries `deletedAt`, and otherwise returns `registration.toDto(state, meta)` (D8, Law 11, E-2, E-24) -- fnList: the handler of `defineList` runs `paginator(ctx.db, contextSchema).query("streams").withIndex("by_identity", (q) => q.eq("tenantId", tenantId).eq("streamType", streamType)).paginate(boundedPage(paginationOpts, { items: limitListPage(registration.mapping.budgetBytes), bytes: limitListBytes }))`, leaves out rows whose `deletedAt` is set unless `includeDeleted` is `true`, reads each remaining row's parts by `by_stream_part` under a derived mapping, maps the rows through `toDto` with a `StreamMeta` built from the row, and returns the helper's result with that `page` in place of the rows; a row saved under a newer `stateSchemaVersion` than the registration's makes `list` throw the plain Error that `load` throws (D8, S4, E-2, E-24, F13) +- fnGet: the handler of `defineGet` reads the stream through the journal's `load`, which reads the row by `by_identity` and assembles a derived state from its parts, returns `null` when the stream does not exist or its row carries `deletedAt`, throws the transient refusal `writePaused` of `spec:command.outcome-boundary` when the row's `stateSchemaVersion` is behind a registered step that reads history, so no DTO is built from a state of the wrong meaning, and otherwise returns `registration.toDto(state, meta)` (D8, Law 11, E-2, E-24, E-25) +- fnList: the handler of `defineList` runs `paginator(ctx.db, contextSchema).query("streams").withIndex("by_identity", (q) => q.eq("tenantId", tenantId).eq("streamType", streamType)).paginate(boundedPage(paginationOpts, { items: limitListPage(registration.mapping.budgetBytes), bytes: limitListBytes }))`, leaves out rows whose `deletedAt` is set unless `includeDeleted` is `true`, reads each remaining row's parts by `by_stream_part` under a derived mapping, maps the rows through `toDto` with a `StreamMeta` built from the row, and returns the helper's result with that `page` in place of the rows; a row saved under a newer `stateSchemaVersion` than the registration's makes `list` throw the plain Error that `load` throws, and a row behind a registered step that reads history makes it throw the transient refusal `writePaused` of `spec:command.outcome-boundary` (D8, S4, E-2, E-24, E-25, F13) - typePageLimit: `type PageLimit = { items: number; bytes: number }` (E-24, F13) - fnBoundedPage: `boundedPage(opts: PaginationOptions, limit: PageLimit): PaginationOptions` returns `{ cursor: opts.cursor, numItems, endCursor: opts.endCursor when set, maximumRowsRead: 2 * limit.items, maximumBytesRead: limit.bytes }` and nothing else of `opts`, where `numItems` is `opts.numItems` floored, raised to 1 when it is not at least 1, a value that is not a number included, and capped at `limit.items`, so `Infinity` asks for the item cap; the context library exports it beside the definers, a parent list imports it from there, and every list, in a context and in the parent, passes its one `paginate` call these options (E-24, F13, Probe 5) - fnHistory: `export const history = query({ args: { tenantId: v.string(), streamId: v.string(), fromVersion: v.optional(v.number()), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(eventEnvelopeValidator), handler })` runs `paginator(ctx.db, contextSchema).query("events").withIndex("by_stream", ...)` with `.eq` on the identity fields and `.gte("streamVersion", fromVersion ?? 1)` in ascending order and paginates with `boundedPage(paginationOpts, { items: limitHistoryPage, bytes: limitHistoryBytes })`; events before a baseline are returned like any other (D5, S4, Sc L2-7, E-24, F13) - componentPagination: `paginator` is imported from `convex-helpers/server/pagination` and called with the component's `ctx.db` and `contextSchema`; it takes the same options as the built-in `.paginate` and returns `{ page, isDone, continueCursor }` with `pageStatus` and `splitCursor` present only when it marks a split; it keeps no journal, so a page is pinned only by an `endCursor`; with an `endCursor` it reads to that cursor and `numItems` no longer bounds it; it counts a row before it tests either cap, so the byte cap is a stopping point that the last row read may pass by its own size; it answers `SplitRequired` when a cap is reached and `SplitRecommended` when a complete page holds more than `numItems + 1` rows (S4, E-24, F13, Probe 5) - pagingByCursor: a client holds one subscription per page, each with `{ cursor, numItems, endCursor }`; the first page starts at `cursor: null`, the next at the previous page's `endCursor`, and the last page is the one whose first read answered `isDone: true`, subscribed with no `endCursor` so that it follows rows added after it (E-24, F15, Probe 5) - fnRebuild: `export const rebuild = query({ args: { tenantId: v.string(), streamId: v.string(), maxEvents: v.optional(v.number()) }, returns: rebuildResultValidator, handler })` calls `rebuildStream` with `maxEvents` and, on a fold, compares with the saved row by deep equality; `maxEvents` is the events the caller may still read in its own transaction, which the restore's stream check passes per stream, and it is lowered to `maxFoldEvents` when larger (D3, D5, Sc L2-7, E-24, E-42) -- typeRebuildResult: `type RebuildResult = { kind: "folded"; state: Value; version: number; fromBaseline: number | null; eventsApplied: number; savedVersion: number; equal: boolean } | { kind: "refused"; reason: "tailTooLong" | "awaitingBaseline" | "budgetExhausted"; tailLength: number; maxFoldEvents: number; savedVersion: number; savedSchemaVersion: number; currentSchemaVersion: number }` where every refusal comes from `rebuildStream` before any event is read; `tailTooLong` and `awaitingBaseline` are what the restore's journal check records as findings, `awaitingBaseline` naming a stream saved under an older meaning that no command or sweep batch has migrated yet, and `budgetExhausted`, returned when the tail exceeds the caller's `maxEvents` but not `maxFoldEvents`, is the point where a batching caller stops and resumes next time, never a finding (E-24, E-25, E-42, F13) +- typeRebuildResult: `type RebuildResult = { kind: "folded"; state: Value; version: number; fromBaseline: number | null; eventsApplied: number; savedVersion: number; equal: boolean } | { kind: "refused"; reason: "tailTooLong" | "awaitingBaseline" | "budgetExhausted"; tailLength: number; maxFoldEvents: number; savedVersion: number; savedSchemaVersion: number; currentSchemaVersion: number }` where every refusal comes from `rebuildStream` before any event is read; `tailTooLong` is what the restore's journal check records as a finding, `awaitingBaseline`, naming a stream saved under an older meaning that no command or sweep batch has migrated yet, is pending work the check inventories and never a finding, because the accepted branch restarts the sweep, and `budgetExhausted`, returned when the tail exceeds the caller's `maxEvents` but not `maxFoldEvents`, is the point where a batching caller stops and resumes next time, never a finding (E-24, E-25, E-42, F13) - validatorRebuildResult: `const rebuildResultValidator = v.union(v.object({ kind: v.literal("folded"), state: v.any(), version: v.number(), fromBaseline: v.union(v.number(), v.null()), eventsApplied: v.number(), savedVersion: v.number(), equal: v.boolean() }), v.object({ kind: v.literal("refused"), reason: v.union(v.literal("tailTooLong"), v.literal("awaitingBaseline"), v.literal("budgetExhausted")), tailLength: v.number(), maxFoldEvents: v.number(), savedVersion: v.number(), savedSchemaVersion: v.number(), currentSchemaVersion: v.number() }))` (E-24, E-25, E-42) - fnByOperation: `export const byOperation = query({ args: { tenantId: v.string(), operationId: v.string() }, returns: v.array(eventEnvelopeValidator), handler })` ranges `by_operation` and collects, for diagnosis and for a receipt's stream versions (D2, D19) - paginationShape: `paginationOptsValidator` from `convex/server` in `args` and `paginationResultValidator(item)` from the same module in `returns`, for every list of the platform, in a context and in the parent; the validator requires `page`, `isDone` and `continueCursor` and admits `pageStatus` and `splitCursor` absent, `null` or set, which covers the helper's result and the built-in's (S4, E-24, Probe 5) diff --git a/design/specs/decisions/d02-context-owns-state-and-journal.sdp.md b/design/specs/decisions/d02-context-owns-state-and-journal.sdp.md index c9ee539..1b7f273 100644 --- a/design/specs/decisions/d02-context-owns-state-and-journal.sdp.md +++ b/design/specs/decisions/d02-context-owns-state-and-journal.sdp.md @@ -20,20 +20,20 @@ Each context is a component that owns its current state, its event journal and i - problem: A central event store with a global position puts a counter in every write's path, and a journal that any parent code can write lets state change without its events; the invalid-transition, stale-version and competing-commands scenarios need the context to be the only writer of its state (Sc L1-1, Sc L1-10, Sc L1-11) - outcome: Each context owns its CMS, journal and stream metadata behind a component boundary, and every event carries the envelope that ties it to its stream, operation, cause and actor (D2) - value: Isolation is the platform's defining property: a context's private schema stays private, cross-context references use application IDs, and no global feed or counter has to exist for the core path (D2) -- risk: Every read of context data from the parent is a component call, and the size of that cost is unknown until the first experiment measures it (D2, F4, Probe 3, OQ1) +- risk: Every read of context data from the parent is a component call, measured by the first experiment on the local backend with every cost target held; its function-call quota is a hosted usage reading (D2, F4, Probe 3, OQ1) - risk: There is no global event order; cross-context history is merged through context APIs (D2) - assumption: Component calls commit or roll back with the calling mutation (F2) - assumption: Nested calls cost more than helper calls, by an amount not yet measured (F4) ### Open questions -- [non-blocking] Probe 3 pending: the cost of one component call in latency and function-call quota decides where hot reads land; isolation itself stays (Probe 3, D2, OQ1) -- [non-blocking] OQ1: whether a trivial context with no invariants of its own may be plain tables in the parent behind lint rules instead of a component is decided after the experiment measures component call cost (OQ1, D2) +- [non-blocking] Probe 3: the latency of one component call is measured and keeps every context a component; its function-call quota is a hosted usage reading, and isolation itself stays (Probe 3, D2, OQ1) +- [non-blocking] OQ1: a trivial context with no invariants of its own stays a component; the owner answered OQ1 on the first experiment's local measurement, and a measured workload that misses an agreed target, or a hosted usage reading above an agreed share, reopens it (OQ1, D2) ## Decision -- context: The concern is ownership of current state, events and stream metadata, and the order and identity of events; Convex gives components whose tables the parent cannot read except through the component's own functions, and gives no global sequence, so a global position would have to be a counter document in every write's path (D2, F2) -- alternative: Do nothing beyond Convex: plain tables in one deployment with a shared journal table any function may write; rejected as the default, because the rule that state never changes without its events cannot be enforced against trusted parent code, though OQ1 keeps it open for trivial contexts (D2, Law 1, Law 2, OQ1, Decision method rule 2) +- context: The concern is ownership of current state, events and stream metadata, and the order and identity of events; Convex gives components whose tables the parent cannot read except through the component's own functions, and gives a commit timestamp that orders committed transactions, whose reader's upper bound is outside the pinned public API, so this design keeps no global position (D2, F2, F18) +- alternative: Do nothing beyond Convex: plain tables in one deployment with a shared journal table any function may write; rejected as the default, because the rule that state never changes without its events cannot be enforced against trusted parent code, and OQ1 is answered with a component for every context (D2, Law 1, Law 2, OQ1, Decision method rule 2) - alternative: A central event-store component with a global position; rejected, because it puts a deployment-wide counter in every write's path, and a global feed comes back only when a real consumer needs one total order (D2, D18) - alternative: One journal library shared as code by every context, each context owning its own tables; this is the option chosen (D2) - decision: Each context is a component that owns its CMS, its event journal and its stream metadata, all written by the context's own mutations; journal code is a shared library each context includes; there is no central event-store component (D2) @@ -42,9 +42,9 @@ Each context is a component that owns its current state, its event journal and i - consequence: Ownership is fixed: CMS, journal events and stream metadata belong to the context, where the metadata may live in the CMS document while enumeration and deletion stay correct; command receipts belong to the parent; access grants belong to the parent or a dedicated access context (D2) - consequence: Every event carries the envelope: `eventId` as stable application identity; `tenantId` and `contextId` as ownership scope; `streamType`, `streamId` and `streamVersion` as exact order within one subject's history; `eventType` and `eventSchemaVersion` as historical contract; `operationId` as the root command or use case that produced it; `correlationId` for diagnostic grouping across operations; `causedBy` as the immediate cause, a command or a triggering event; `actor` as the server-established actor kind, identity and delegation reference; `recordedAt` and optional `occurredAt`; `payload` as validated, bounded fact data (D2) - consequence: An append names the stream version it expects; the check is an indexed read in the same transaction, because an index alone is not a unique constraint (D2, F1) -- consequence: Order exists only within a stream; no deployment-wide order token exists, and timestamps, `_creationTime` and UUID order never prove that a consumer has seen every committed event (D2) +- consequence: Order within the platform exists only within a stream; Convex documents a commit timestamp that orders committed transactions across the deployment, which orders transactions and not the events of one, so `_creationTime`, `recordedAt` and UUID order never prove that a consumer has seen every committed event, and whether an ordered consumer takes the commit timestamp as its position is decided with D18's consumer (D2, D18, F18) - consequence: Events in different streams relate through `operationId` and `causedBy`, and a receipt may list the stream versions one operation touched (D2) - consequence: A reference that crosses a context boundary is `(tenantId, contextId, eventId)`, never a private document ID; context APIs return DTOs and application IDs (D2) - consequence: The starting indexes are stream replay, event identity and stream enumeration; others are added when a real query needs them (D2) -- consequence: There is no global event order; cross-context history is merged through context APIs, and a global feed returns only when a real consumer needs one total order (D2, D18) +- consequence: There is no global event order; cross-context history is merged through context APIs, and a global feed returns only when a real consumer needs one total order, with the commit timestamp as the candidate token (D2, D18, F18) - consequence: The standing cost is one component per context with its own tables and functions, and one component call per parent read of context data, whose cost the first experiment measures (D2, F4, Probe 3) diff --git a/design/specs/decisions/d13-deferred-work-is-an-obligation.sdp.md b/design/specs/decisions/d13-deferred-work-is-an-obligation.sdp.md index 1bf8222..5387f22 100644 --- a/design/specs/decisions/d13-deferred-work-is-an-obligation.sdp.md +++ b/design/specs/decisions/d13-deferred-work-is-an-obligation.sdp.md @@ -32,7 +32,7 @@ An obligation records one promise. It commits in the same transaction as the bus ### Open questions -- [non-blocking] Probe 7 observes the local scheduled reaction, states, scan boundary and both CLI replacement modes; the owner must apply those measurements to the do-nothing check before activation; seven-day expiry, hosted dashboard restore, Workpool and Workflow remain unprobed (Probe 7, D13, F12, F16) +- [non-blocking] Probe 7 observes the local scheduled reaction, states, scan boundary and both CLI replacement modes; the owner confirmed D13 on these observations; seven-day expiry, hosted dashboard restore, Workpool and Workflow remain unprobed (Probe 7, D13, F12, F16) ## Decision diff --git a/design/specs/facts/f18-commit-timestamps.sdp.md b/design/specs/facts/f18-commit-timestamps.sdp.md index 9d3699c..6f786be 100644 --- a/design/specs/facts/f18-commit-timestamps.sdp.md +++ b/design/specs/facts/f18-commit-timestamps.sdp.md @@ -10,7 +10,7 @@ relations: F18 · Status: probed · Doc status: not in the doc's ledger · Decisions: D2, D18. -The pinned package declares `db.vars.commitTs`, a placeholder that a mutation writes into a field and that resolves at commit to a bigint ordered by commit order. Probe 8 ran it on the native backend, release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0: one transaction gets one timestamp in the parent and in two components, and later transactions get larger ones. D2 cites the doc's sentence that no deployment-wide order token exists, and whether this fact changes that sentence is the owner's. A timestamp is shared by every row one transaction writes, so it orders transactions and not the events inside one. A reader's upper bound, `getSnapshotTs()`, is marked internal in the package source and absent from its public declarations, so it is no public promise of the pinned release. +The pinned package declares `db.vars.commitTs`, a placeholder that a mutation writes into a field and that resolves at commit to a bigint ordered by commit order. Probe 8 ran it on the native backend, release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0: one transaction gets one timestamp in the parent and in two components, and later transactions get larger ones. D2 cites the doc's sentence that no deployment-wide order token exists, and whether this fact changes that sentence is the owner's. A timestamp is shared by every row one transaction writes, so it orders transactions and not the events inside one. The documentation describes `ctx.meta.getSnapshotTs()` as the reader's upper bound, the snapshot timestamp on the commit clock, and the public declarations of the pinned package omit it, so the bound is unavailable through the pinned public API and nothing here uses it. ## Intent @@ -18,14 +18,14 @@ The pinned package declares `db.vars.commitTs`, a placeholder that a mutation wr ### Open questions -- [non-blocking] The internal runtime upper-bound method is absent from the public server declarations. A supported public upper bound, ordering under overlapping transactions and behavior on a hosted deployment are outside the native example; no platform consumer adopts the internal method by this fact (F18, Probe 8) +- [non-blocking] The documented upper bound, `ctx.meta.getSnapshotTs()`, is absent from the pinned package's public declarations; ordering under overlapping transactions and behavior on a hosted deployment are outside the native example; a platform consumer adopts the bound only through a public declaration of a pinned package (F18, Probe 8) ## Constraints - statement: On the pinned native backend, a transaction writes the same resolved bigint commit timestamp in the parent and two components, later sequential transactions have larger timestamps, and a top-level mutation may return its placeholder as the resolved bigint; within the mutation a read yields the unresolved placeholder and scheduling that placeholder is refused (F18) - flavor: convex-fact - target: evidence.status:probed -- measurableBy: Compiled tier: `convex` 1.46.0, `node_modules/convex/dist/esm-types/server/database.d.ts:308-318` declares the placeholder and `:199` keeps the schema type on insert, and `node_modules/convex/dist/esm-types/server/meta.d.ts:125-136` declares no snapshot timestamp method; package source `node_modules/convex/src/server/meta.ts:137-160` describes the internal `getSnapshotTs()` upper bound, and `node_modules/convex/src/server/impl/meta_impl.ts:56-59,78,94` supplies it at runtime; `node_modules/convex/src/values/value.ts:98-127` defines unresolved numeric conversion; native backend tier: Probe 8 on the pinned release (F18) +- measurableBy: Compiled tier: `convex` 1.46.0, `node_modules/convex/dist/esm-types/server/database.d.ts:308-318` declares the placeholder and `:199` keeps the schema type on insert, and `node_modules/convex/dist/esm-types/server/meta.d.ts:125-136` declares no snapshot timestamp method, while the page `docs.convex.dev/database/advanced/commit-timestamp`, section Snapshot timestamp, documents it; package source `node_modules/convex/src/server/meta.ts:137-160` describes the internal `getSnapshotTs()` upper bound, and `node_modules/convex/src/server/impl/meta_impl.ts:56-59,78,94` supplies it at runtime; `node_modules/convex/src/values/value.ts:98-127` defines unresolved numeric conversion; native backend tier: Probe 8 on the pinned release (F18) ## Design diff --git a/design/specs/obligations/do-nothing-check.sdp.md b/design/specs/obligations/do-nothing-check.sdp.md index 6058f91..41fc9b9 100644 --- a/design/specs/obligations/do-nothing-check.sdp.md +++ b/design/specs/obligations/do-nothing-check.sdp.md @@ -30,7 +30,7 @@ Before the obligation module is built, the do-nothing option is checked against ### Open questions -- [non-blocking] Probe 7 observes a plain scheduled local reaction and all five states on the pinned native backend; a failed-function scan counts zero documents and bytes but exhausts the 4096-read allowance at 4092 total rows; fresh CLI replacement leaves no scheduler intent, while replacement in place preserves later intent outside restored data; seven-day expiry, hosted dashboard restore, Workpool and Workflow remain open, and the owner rules on D13 from these observations (Probe 7, F12, F16, F13) +- [non-blocking] Probe 7 observes a plain scheduled local reaction and all five states on the pinned native backend; a failed-function scan counts zero documents and bytes but exhausts the 4096-read allowance at 4092 total rows; fresh CLI replacement leaves no scheduler intent, while replacement in place preserves later intent outside restored data; seven-day expiry, hosted dashboard restore, Workpool and Workflow remain open, and the owner confirmed D13 on these observations (Probe 7, F12, F16, F13) ## Decision @@ -42,5 +42,5 @@ Before the obligation module is built, the do-nothing option is checked against - rationale: Only a failing scenario justifies a mechanism, and the cheapest mechanism that passes wins; the four reasons are each a scenario the plain scheduled mutation fails (Decision method rule 3, D13) - rationale: The system table is read-only metadata for 7 days; it cannot hold business evidence, a stored authority, or an operator's repair (F16, D13) - consequence: Every local reaction of the durable profile is an obligation and ships the module's recovery, inspection and retention, as Law 8 requires of every installed durable capability; the plain scheduled mutations that remain are the platform's own maintenance batches, rebuild, baseline migration and restore checks, which are operator operations and not deferred business work (D13, Law 8) -- consequence: The probe result is recorded on `spec:facts.f16-scheduled-functions-table-shows-failed-runs` and on `spec:facts.f12-backups-exclude-pending-scheduled-functions`, and this Spec's readiness is restated by the owner after it (Probe 7, Fact ledger) +- consequence: The probe result is recorded on `spec:facts.f16-scheduled-functions-table-shows-failed-runs` and on `spec:facts.f12-backups-exclude-pending-scheduled-functions`, and the owner confirmed the decision on them (Probe 7, Fact ledger) - consequence: The standing cost when the profile is activated is the one D13 lists; while the durable profile is not installed, or if Probe 7 lets the do-nothing option stand for the profile, the cost is the scan of the system table, bounded by the scanned-documents ceiling and measured by the probe (D13, F13) diff --git a/design/specs/obligations/obligation-module.sdp.md b/design/specs/obligations/obligation-module.sdp.md index 7d0378e..5e08552 100644 --- a/design/specs/obligations/obligation-module.sdp.md +++ b/design/specs/obligations/obligation-module.sdp.md @@ -50,9 +50,9 @@ The module lives in the parent as a shared library and two parent tables, becaus ### Open questions -- [non-blocking] Probe 7 observes local scheduler states, short retention, both CLI replacement modes, scheduler-ID validation and the failed-function scan boundary; a stored `v.id("_scheduled_functions")` can validate after fresh replacement while resolving to no row, so it is a typed reference and not proof of a live dispatch; seven-day expiry, hosted restore, Workpool and Workflow remain open, and the owner rules on the do-nothing check (Probe 7, D13, F12, F16) +- [non-blocking] Probe 7 observes local scheduler states, short retention, both CLI replacement modes, scheduler-ID validation and the failed-function scan boundary; a stored `v.id("_scheduled_functions")` can validate after fresh replacement while resolving to no row, so it is a typed reference and not proof of a live dispatch; seven-day expiry, hosted restore, Workpool and Workflow remain open, and the owner confirmed the do-nothing check's rejection on these observations (Probe 7, D13, F12, F16) - [non-blocking] Extension E-50: the doc does not say where the module lives; this design places it in the parent as a library plus the `obligations` and `obligationRepairs` tables with a static handler registry keyed by handler key and version, because the wrapper runs parent-side bodies as nested mutations and the record reuses the parent's actor and scope types; the alternative, a Convex component driven by function handles like Workpool, is not pre-empted (D13, D11, E-50) -- [non-blocking] OQ1 dependency: if the first experiment shows that a trivial context may be plain tables, a local reaction's body still runs as a nested mutation and nothing here changes; if component call cost breaks the read budgets, only the bodies change shape (OQ1, Probe 3) +- [non-blocking] OQ1 dependency: OQ1 is answered with a component for every context, so a local reaction's body runs as a nested mutation; if a measured workload reopens it, only the bodies change shape (OQ1, Probe 3) - [non-blocking] Decision method rule 4: the doc says Layers 0 to 2 get full detail, the first experiment runs, and Layer 3 is written from what it showed; this module and its children are written now, at the depth D13 to D15 give, with numeric defaults the doc does not give, the backoff and attempt budget of E-51, the sweeper's cadence and bounds of E-52, the fan-out bounds of E-54, the retention horizons of E-55, the lease and settle repeats of E-58 and the index set of E-57; every such number is a provisional default until the experiment's measurements and Probe 7 confirm or replace it, and the owner confirms that writing the structure now does not pre-empt the experiment (Decision method rule 4, D13, Probe 7, E-51, E-52, E-54, E-55, E-57, E-58) ## Behavior diff --git a/design/specs/platform/transactional-domain-platform.sdp.md b/design/specs/platform/transactional-domain-platform.sdp.md index a1f9997..2a76fb9 100644 --- a/design/specs/platform/transactional-domain-platform.sdp.md +++ b/design/specs/platform/transactional-domain-platform.sdp.md @@ -34,14 +34,14 @@ Citations in this corpus use the tokens the plan fixes: D for a decision, Law fo - problem: Adding a domain capability to today's platform adds a queue, a projection job, a registry or a governance table before it adds a business rule, and each of those then needs its own recovery, inspection and retention; a maintainer cannot trace one command's success or failure without unrelated machinery (Thesis, Decision method rule 7) - outcome: Adding domain sophistication does not add infrastructure: one mutation per business operation, contexts that own their state and journal, and layers each usable before the next exists (Thesis) - value: A new feature needs only its domain input, events and decision, one command or use-case binding, and the read model it actually uses; a maintainer traces its success and failure without unrelated queue, agent or governance registries (Thesis) -- risk: Every read of context data from the parent is a component call whose cost the first experiment must measure; if it breaks the read budgets, the shape of the context component changes before Layer 3 is designed (D2, Probe 3, OQ1) +- risk: Every read of context data from the parent is a component call, whose cost the first experiment measured on the local backend and which keeps every context a component; a measured workload that misses an agreed target, or a hosted usage reading above an agreed share, changes the shape of the context component before Layer 3 is designed (D2, Probe 3, OQ1) - risk: Nothing has run yet; the sources support the Convex facts, not the correctness or performance of the design (Sources) - assumption: Mutations are serializable under optimistic concurrency, and component calls commit or roll back with the calling mutation (F1, F2) - assumption: One ledger fact is still assumed, and three that the doc lists as assumed are rechecked or probed in the corpus; each Spec that rests on one names it and the probe plan names the probe (F14, F15, F16, F17) ### Open questions -- [non-blocking] OQ5 asks whether the decisions move into SDP carriers; this corpus is that move, with ledger facts as constraint Specs, decisions as decision Specs that list the do-nothing option first, and scenarios as example Specs under the component that provides the capability; the owner confirms the mapping or chooses rule Specs for facts, which is a mechanical change (OQ5) +- [non-blocking] OQ5 asks whether the decisions move into SDP carriers; this corpus is that move, with ledger facts as constraint Specs, decisions as decision Specs that list the do-nothing option first, and scenarios as example Specs under the component that provides the capability, which is the form the doc's behavior rules with examples take here; the owner confirmed the mapping (OQ5) ## Behavior diff --git a/design/specs/platform/vocabulary.sdp.md b/design/specs/platform/vocabulary.sdp.md index eee6872..ff9a9cc 100644 --- a/design/specs/platform/vocabulary.sdp.md +++ b/design/specs/platform/vocabulary.sdp.md @@ -10,20 +10,20 @@ relations: Feature · Detail: full transcription · Traces: Vocabulary, D2, D3, D4, D6, D8, D9, D11, D13, OQ6. -The doc fixes nine terms and names two clashes with today's glossary. This model carries those nine, the words the doc uses without defining them separately, and the terms the other packages need so that every family names one concept with one word. Where a Convex term and a doc term differ, the doc's word names the design concept and the Convex word names the mechanism: an obligation is the record, a scheduled function is the dispatch. Stream version, never revision. Obligation for the record, effect for the change it makes, attempt for one try. Receipt for the stored idempotency outcome. Generation for a numbered read-model build. +`CONTEXT.md` is the platform's language and this model follows it: where the two differ, `CONTEXT.md`'s word wins and this list is corrected. The doc fixes nine terms and names two clashes with today's glossary. This model carries those nine, the words the doc uses without defining them separately, and the terms the other packages need so that every family names one concept with one word. Where a Convex term and a doc term differ, the doc's word names the design concept and the Convex word names the mechanism: an obligation is the record, a scheduled function is the dispatch. Stream version, never revision. Obligation for the record, effect for the change it makes, attempt for one try. Receipt for the stored idempotency outcome. Generation for a numbered read-model build. -The two clashes stay open for the owner. The corpus proceeds on the provisional reading recorded in the open questions: a row that proves an effect happened is completion evidence, and the attempt number of watched work is the attempt. +The two clashes are answered: a row that proves an effect happened is completion evidence, and the attempt number of watched work is the attempt. ## Intent -- outcome: One word per concept across every package of the corpus, with the doc's meaning where the doc fixes one and the two glossary clashes recorded rather than resolved (Vocabulary, OQ6) +- outcome: One word per concept across every package of the corpus, with the doc's meaning where the doc fixes one and `CONTEXT.md`'s word where the two glossary clashes were answered (Vocabulary, OQ6) - value: Specs, code and tests name the same things the same way, and a reviewer can reject a synonym for a locked term by pointing at one list (Vocabulary) -- risk: Two terms clash with today's glossary; until the owner picks one meaning each, code must not name a receipt or a generation (OQ6) +- risk: A word used for two things is a gap in the language; a reviewer reports it against `CONTEXT.md`, and the owner rules on the term (OQ6) ### Open questions -- [non-blocking] OQ6, receipt: this design uses receipt for the stored outcome of a command for idempotency, while today's glossary uses receipt for a row that proves an effect happened; the corpus reads the latter as completion evidence and the owner picks one meaning before code names it (OQ6, Vocabulary) -- [non-blocking] OQ6, generation: this design uses generation for a numbered build of a read model, while today's glossary uses it for the attempt number of watched work; the corpus reads the latter as attempt and the owner picks one meaning before code names it (OQ6, Vocabulary) +- [non-blocking] OQ6, receipt: this design uses receipt for the stored outcome of a command for idempotency, while today's glossary uses receipt for a row that proves an effect happened; the owner answered OQ6 with receipt for the stored outcome of a command and completion evidence for the row that proves an effect happened (OQ6, Vocabulary) +- [non-blocking] OQ6, generation: this design uses generation for a numbered build of a read model, while today's glossary uses it for the attempt number of watched work; the owner answered OQ6 with generation for a numbered build of a read model and attempt for one try at an obligation (OQ6, Vocabulary) ## Model @@ -34,8 +34,12 @@ The two clashes stay open for the owner. The corpus proceeds on the provisional - **event** — A recorded fact in a stream's history, carried in the fifteen-field envelope that D2 fixes (D2). - **envelope** — The fifteen fields every event carries: identity, ownership scope, position in the stream, historical contract, operation, correlation, cause, actor, times and payload (D2). - **baseline event** — An event holding migrated state, from which rebuild starts; earlier events stay as readable history (Vocabulary, D5). -- **command** — A request to change one context's state, evaluated by `decide` against current state; commands load current state and apply only new events (D3). -- **decider** — The pure domain code of a context: `decide`, `evolve` and the initial state, reading no database, network, scheduler, environment or ambient auth (D3). +- **command** — A caller's request to the parent to carry out one business operation, with one declaration and exactly one outcome; a stream command is the part addressed to one stream, which that stream's decider decides against current state, loading current state and applying only new events (D3, D10). +- **entry** — A function the parent registers for a caller: a public command, an internal command, a parent query or an operator entry (D7, D19). +- **parent query** — A read the parent answers to a caller, authorized in the parent, relaying a context query or reading a read model (D8, D11). +- **context query** — A read a context answers to the parent, returning a DTO (D8). +- **query refusal** — The refusal of a parent query, in a rejection's wire shape with the name of the entry that refused; it is not an outcome and nothing is recorded (D7, Law 5). +- **decider** — The pure rules of one stream type: `decide`, `evolve` and the initial state, reading no database, network, scheduler, environment or ambient auth (D3). - **fold** — Computing the next state as `fold(evolve, state, newEvents)`; the one authority for state (D3). - **outcome** — One of applied, business failure, rejection or technical failure (D4). - **applied** — The outcome where events were recorded and state changed (D4). @@ -58,6 +62,19 @@ The two clashes stay open for the owner. The corpus proceeds on the provisional - **projection** — The named, versioned, deterministic logic that maintains a read model; live update and rebuild share it (D8). - **DTO** — The deliberate shape a context query returns; the context's private schema stays private (D8). - **generation** — A numbered build of a read model (Vocabulary, D9). +- **installation** — Making a read model exist in a deployment: its declaration and its first rebuild (D9). +- **backfill** — The part of a rebuild that writes a generation's rows from its sources (D9). +- **verification** — The part of a rebuild that checks a generation's rows against its sources and corrects them (D9). +- **pass** — One traversal of a rebuild over its sources or its rows: a backfill, a verification, a purge or a fill (D9). +- **switch** — Making a verified generation the one readers see (D9). +- **rollback** — Making a retired generation the one readers see again (D9). +- **retired generation** — A generation readers no longer see, kept until it is purged (D9). +- **purge** — Deleting a retired generation's rows (D9). +- **fence** — The number a generation's batches carry so that a stale batch writes nothing (D9). +- **progress row** — The row that holds where a rebuild stands, read by its batches and by no command (D9). +- **tenant list** — The parent's list of tenants, which a rebuild and a sweep walk (D11, D19). +- **backup archive** — The exported file of a deployment's table data at one moment, which a restore imports; a snapshot is a transaction's view of the database, never the archive (D19). +- **stated operator** — The name an administrator states at an operator entry; it records who said they acted and establishes no actor (D19, D11). - **write pause** — The maintenance gate under which a read model that depends on history across several streams rebuilds; every writer in scope reads it (D9). - **obligation** — The durable record of promised deferred work; one promise, committed with the business change (Vocabulary, D13). - **effect** — The change a piece of deferred work makes, as distinct from the obligation that records the promise (Vocabulary, D13). diff --git a/package.json b/package.json index 8f5d56c..75bf6df 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "libar-platform", "version": "0.0.0", "private": true, + "license": "Apache-2.0", "type": "module", "description": "The Convex transactional domain platform: its design as a Software Delivery Protocol corpus, and the code built from it.", "engines": { diff --git a/tests/pure/acceptance.test.ts b/tests/pure/acceptance.test.ts index d1d6ff7..88dc061 100644 --- a/tests/pure/acceptance.test.ts +++ b/tests/pure/acceptance.test.ts @@ -810,7 +810,8 @@ test("pure: acceptanceRows names the rows the doc's table gives Layer 0, 1 or 2, } const graph = loadGraph(join(root, "generated/graph.json")); const rows = requiredRows(graph, firstExperimentSpec); - expect(rows).toHaveLength(23); + tableRows.push("Sc ALL-1"); + expect(rows).toHaveLength(24); expect(rows).toEqual(tableRows); const scenarios = deriveRequiredScenarios(graph, firstExperimentSpec); expect( From 3e74cd387ce15ee2257955622b9af65f915e8122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:29:48 +0200 Subject: [PATCH 112/155] build: 4d the receipts sweep --- example/convex/_generated/api.d.ts | 2 + example/convex/receipts.ts | 13 ++ fixture/convex/_generated/api.d.ts | 2 + fixture/convex/receipts.ts | 13 ++ src/command/receipts.ts | 74 +++++++++ tests/simulator/receipts-sweep.test.ts | 213 +++++++++++++++++++++++++ 6 files changed, 317 insertions(+) create mode 100644 example/convex/receipts.ts create mode 100644 fixture/convex/receipts.ts create mode 100644 tests/simulator/receipts-sweep.test.ts diff --git a/example/convex/_generated/api.d.ts b/example/convex/_generated/api.d.ts index 4a7dc62..313b8f5 100644 --- a/example/convex/_generated/api.d.ts +++ b/example/convex/_generated/api.d.ts @@ -15,6 +15,7 @@ import type * as orderSummary from "../orderSummary.js"; import type * as ordering from "../ordering.js"; import type * as readModels from "../readModels.js"; import type * as rebuild from "../rebuild.js"; +import type * as receipts from "../receipts.js"; import type * as receiving from "../receiving.js"; import type { @@ -31,6 +32,7 @@ declare const fullApi: ApiFromModules<{ ordering: typeof ordering; readModels: typeof readModels; rebuild: typeof rebuild; + receipts: typeof receipts; receiving: typeof receiving; }>; diff --git a/example/convex/receipts.ts b/example/convex/receipts.ts new file mode 100644 index 0000000..590aac1 --- /dev/null +++ b/example/convex/receipts.ts @@ -0,0 +1,13 @@ +// The composition's receipt sweep, called by operations. +import * as receipts from "../../src/command/receipts.js"; +import { internalMutation } from "./_generated/server.js"; +export const sweep = internalMutation({ + args: receipts.sweepArgs, + returns: receipts.sweepResultValidator, + handler: receipts.sweep, +}); +export const sweepNext = internalMutation({ + args: receipts.sweepNextArgs, + returns: receipts.sweepNextResultValidator, + handler: receipts.sweepNext, +}); diff --git a/fixture/convex/_generated/api.d.ts b/fixture/convex/_generated/api.d.ts index 66614f2..ba51e5c 100644 --- a/fixture/convex/_generated/api.d.ts +++ b/fixture/convex/_generated/api.d.ts @@ -30,6 +30,7 @@ import type * as parentList from "../parentList.js"; import type * as readCost from "../readCost.js"; import type * as readModels from "../readModels.js"; import type * as rebuild from "../rebuild.js"; +import type * as receipts from "../receipts.js"; import type * as rejectionCommands from "../rejectionCommands.js"; import type * as scheduledRows from "../scheduledRows.js"; import type * as summaries from "../summaries.js"; @@ -67,6 +68,7 @@ declare const fullApi: ApiFromModules<{ readCost: typeof readCost; readModels: typeof readModels; rebuild: typeof rebuild; + receipts: typeof receipts; rejectionCommands: typeof rejectionCommands; scheduledRows: typeof scheduledRows; summaries: typeof summaries; diff --git a/fixture/convex/receipts.ts b/fixture/convex/receipts.ts new file mode 100644 index 0000000..590aac1 --- /dev/null +++ b/fixture/convex/receipts.ts @@ -0,0 +1,13 @@ +// The composition's receipt sweep, called by operations. +import * as receipts from "../../src/command/receipts.js"; +import { internalMutation } from "./_generated/server.js"; +export const sweep = internalMutation({ + args: receipts.sweepArgs, + returns: receipts.sweepResultValidator, + handler: receipts.sweep, +}); +export const sweepNext = internalMutation({ + args: receipts.sweepNextArgs, + returns: receipts.sweepNextResultValidator, + handler: receipts.sweepNext, +}); diff --git a/src/command/receipts.ts b/src/command/receipts.ts index fa0ffa0..25bbdf6 100644 --- a/src/command/receipts.ts +++ b/src/command/receipts.ts @@ -4,6 +4,8 @@ import { convexToJson, type JSONValue, type Value } from "convex/values"; import type { AffectedRef, StreamVersion } from "../kernel/index.js"; import type { CallerNamespace } from "./actor-and-scope.js"; import type { MutationCtx, Receipt } from "./tables.js"; +import { nextTenant } from "./authority.js"; +import { v } from "convex/values"; export type ReceiptKey = { tenantId: string; namespace: CallerNamespace; @@ -114,3 +116,75 @@ export async function insertReceipt( tombstone: false, }); } + +// spec:command.receipt-table fnSweep. Operations supplies the time and batch size. +export const limitSweepBatch = 1000; +export const sweepArgs = { + tenantId: v.string(), + now: v.number(), + limit: v.number(), +}; +export const sweepResultValidator = v.object({ + deleted: v.number(), + compacted: v.number(), + more: v.boolean(), +}); +export type SweepResult = { deleted: number; compacted: number; more: boolean }; +export async function sweep( + ctx: MutationCtx, + { tenantId, now, limit }: { tenantId: string; now: number; limit: number }, +): Promise { + if (!Number.isFinite(now)) + throw new Error("Receipt sweep now must be finite"); + if (!Number.isSafeInteger(limit) || limit < 1) + throw new Error("Receipt sweep limit must be a positive safe integer"); + const size = Math.min(limit, limitSweepBatch); + const expired = () => + ctx.db + .query("receipts") + .withIndex("by_tenant_expiry", (q) => + q.eq("tenantId", tenantId).lte("expiresAt", now), + ); + const rows = await expired().take(size); + let deleted = 0; + for (const row of rows) { + // The command path has no tombstone retention or replay. Refuse rather than destroy one. + if (row.tombstone) + throw new Error( + "Receipt sweep cannot handle tombstones until lookup supports them", + ); + const classified = classifyReceipt( + row, + row.fingerprint, + row.contractVersion, + now, + ); + if (classified.class !== "new") + throw new Error("Receipt sweep expected an expired receipt"); + await ctx.db.delete(row._id); + deleted++; + } + return { deleted, compacted: 0, more: (await expired().first()) !== null }; +} +// The operations caller carries after between runs. A tenant with more stays on the same cursor; +// a null tenant ends the pass. Nothing schedules another run. +export const sweepNextArgs = { + after: v.union(v.string(), v.null()), + now: v.number(), + limit: v.number(), +}; +export const sweepNextResultValidator = v.object({ + ...sweepResultValidator.fields, + tenantId: v.union(v.string(), v.null()), + after: v.union(v.string(), v.null()), +}); +export async function sweepNext( + ctx: MutationCtx, + args: { after: string | null; now: number; limit: number }, +): Promise { + const tenantId = await nextTenant(ctx, args.after); + if (tenantId === null) + return { tenantId, after: null, deleted: 0, compacted: 0, more: false }; + const result = await sweep(ctx, { ...args, tenantId }); + return { ...result, tenantId, after: result.more ? args.after : tenantId }; +} diff --git a/tests/simulator/receipts-sweep.test.ts b/tests/simulator/receipts-sweep.test.ts new file mode 100644 index 0000000..0043576 --- /dev/null +++ b/tests/simulator/receipts-sweep.test.ts @@ -0,0 +1,213 @@ +import { convexTest, type TestConvex } from "convex-test"; +import type { SchemaDefinition } from "convex/server"; +import { commandTables } from "../../src/command/tables.js"; +import { expect, test } from "vitest"; +import schema from "../../fixture/convex/schema.js"; +import { internal as fixtureInternal } from "../../fixture/convex/_generated/api.js"; +import { internal as productionInternal } from "../../example/convex/_generated/api.js"; +import { productionTest } from "./production.js"; + +const compositions: { + name: string; + app: () => TestConvex>; + internal: typeof fixtureInternal | typeof productionInternal; +}[] = [ + { + name: "fixture", + app: () => + convexTest(schema, import.meta.glob("../../fixture/convex/**/*.ts")), + internal: fixtureInternal, + }, + { name: "production", app: productionTest, internal: productionInternal }, +]; +const now = 1000; +function receipt(tenantId: string, requestKey: string, expiresAt = now) { + return { + tenantId, + requestKey, + expiresAt, + namespace: "worker" as const, + commandType: "SweepTest", + fingerprint: "fingerprint", + contractVersion: 1, + outcome: "applied" as const, + operationId: requestKey, + affected: [ + { contextId: "context", streamType: "subject", streamId: requestKey }, + ], + versions: [ + { + tenantId, + contextId: "context", + streamType: "subject", + streamId: requestKey, + version: 1, + }, + ], + actorId: "worker", + recordedAt: 0, + tombstone: false, + }; +} +for (const composition of compositions) { + // spec:command.idempotency-and-receipts limitSweepBatch; spec:command.receipt-table fnSweep. + test.each([3, 1000, 1001])( + `convex-test: ${composition.name} sweep bounds expired receipts at %i`, + async (count) => { + const t = composition.app(); + const kept = await t.run(async (ctx) => { + for (let i = 0; i < count; i++) + await ctx.db.insert("receipts", receipt("a", `expired-${i}`)); + const future = await ctx.db.insert( + "receipts", + receipt("a", "future", now + 1), + ); + const other = await ctx.db.insert("receipts", receipt("b", "other")); + return [await ctx.db.get(future), await ctx.db.get(other)]; + }); + expect( + await t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 2000, + }), + ).toEqual({ + deleted: Math.min(count, 1000), + compacted: 0, + more: count > 1000, + }); + const remaining = await t.run((ctx) => + ctx.db.query("receipts").collect(), + ); + for (const row of kept) expect(remaining).toContainEqual(row); + expect(remaining).toHaveLength(Math.max(0, count - 1000) + 2); + if (count > 1000) + expect( + await t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 1000, + }), + ).toEqual({ deleted: 1, compacted: 0, more: false }); + expect( + await t.run((ctx) => + ctx.db.system.query("_scheduled_functions").collect(), + ), + ).toEqual([]); + }, + ); + test(`convex-test: ${composition.name} sweep honors a smaller limit and refuses invalid limits`, async () => { + const t = composition.app(); + await t.run(async (ctx) => { + for (let i = 0; i < 3; i++) + await ctx.db.insert("receipts", receipt("a", `${i}`)); + }); + for (const limit of [0, -1, 1.5]) + await expect( + t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit, + }), + ).rejects.toThrow("positive safe integer"); + expect( + await t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 2, + }), + ).toEqual({ deleted: 2, compacted: 0, more: true }); + }); + // The absent irreversible retention path is a gap against the Spec's expiry narrative. + test(`convex-test: ${composition.name} refuses a tombstone and rolls back all sweep deletes`, async () => { + const t = composition.app(); + await t.run(async (ctx) => { + await ctx.db.insert("receipts", receipt("a", "reversible", now - 1)); + await ctx.db.insert("receipts", { + ...receipt("a", "irreversible"), + tombstone: true, + affected: [], + versions: [], + }); + }); + const before = await t.run((ctx) => ctx.db.query("receipts").collect()); + await expect( + t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 1000, + }), + ).rejects.toThrow("until lookup supports them"); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual( + before, + ); + }); + // spec:command.actor-and-scope fnNextTenant and the receipt sweep's per-tenant boundary. + test(`convex-test: ${composition.name} sweep loop visits one tenant per run and finishes the current tenant first`, async () => { + const t = composition.app(); + await t.run(async (ctx) => { + for (const tenantId of ["b", "a"]) { + await ctx.db.insert("tenants", { tenantId, createdAt: 0 }); + for (let i = 0; i < 2; i++) + await ctx.db.insert( + "receipts", + receipt(tenantId, `${tenantId}-${i}`), + ); + } + }); + const run = (after: string | null) => + t.mutation(composition.internal.receipts.sweepNext, { + after, + now, + limit: 1, + }); + expect(await run(null)).toEqual({ + tenantId: "a", + after: null, + deleted: 1, + compacted: 0, + more: true, + }); + expect( + await t.run((ctx) => + ctx.db + .query("receipts") + .withIndex("by_tenant_expiry", (q) => q.eq("tenantId", "b")) + .collect(), + ), + ).toHaveLength(2); + expect(await run(null)).toEqual({ + tenantId: "a", + after: "a", + deleted: 1, + compacted: 0, + more: false, + }); + expect(await run("a")).toEqual({ + tenantId: "b", + after: "a", + deleted: 1, + compacted: 0, + more: true, + }); + expect(await run("a")).toEqual({ + tenantId: "b", + after: "b", + deleted: 1, + compacted: 0, + more: false, + }); + expect(await run("b")).toEqual({ + tenantId: null, + after: null, + deleted: 0, + compacted: 0, + more: false, + }); + expect( + await t.run((ctx) => + ctx.db.system.query("_scheduled_functions").collect(), + ), + ).toEqual([]); + }); +} From 19738b49831c0624d94a30c5cda534a5547a5304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:31:22 +0200 Subject: [PATCH 113/155] design: the close of round 3, OD-149 decided by its checked lean, the checks of OD-095 and OD-135 recorded Co-Authored-By: Claude Opus 5.5 --- design/decisions/register.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/design/decisions/register.json b/design/decisions/register.json index 1e7a12b..e7c3f11 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -3855,7 +3855,7 @@ "lean": { "do": "Keep the 7-day default retry window, expiry checked at lookup, the per-tenant sweep of at most 1,000, the 256-byte key and tenant bounds and the 1,000 affected refs." }, - "checkedBy": null, + "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: every checked claim held, 6 of 6, at idempotency-and-receipts.sdp.md:69,83,99,100,101,102 (job-mus2kbol-a1d74933, plant caught); the lean has only its do, so it waits for its advisor's lean in full before it is applied", "fork": null, "decidedBy": null, "decidedOn": null, @@ -5627,7 +5627,7 @@ "changes": "For the lean: nothing in `spec:operations.baseline-operations`; the pipeline's step 10 gains `writeAudit` for declarations whose `audit` field is set [design/specs/operations/baseline-operations.sdp.md:66], which `spec:command.command-declaration` must carry. For the alternative, audit as events in a context or one table per kind: [design/specs/operations/baseline-operations.sdp.md:71] to [design/specs/operations/baseline-operations.sdp.md:83] and the restore's receipt check that reads audit by operation [design/specs/operations/baseline-operations.sdp.md:79].", "ifWrong": "A wrong table shape is an edit and a migration of an operational table, never of history; a wrong failure rule is found by the fault-injection scenario before production data; **Judgment.** a missing bound on `details` costs a failed command that the operator finds by the diagnostic record's error code, and a bound is one edit." }, - "checkedBy": null, + "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: 9 of 19 claims held; the do holds at corrected lines (baseline-operations.sdp.md:72,81,83,85,88,93,94,95,96; d19:40,43), but the against's unbounded details field does not exist (the audit schema at :78 has none) and 'nothing is built' is stale (src/audit/write.ts, tests/simulator/audit.test.ts); the lean needs its advisor's correction before it is applied (job-mus2kbol-a1d74933, plant caught)", "fork": null, "decidedBy": null, "decidedOn": null, @@ -6140,7 +6140,7 @@ "provisionalReading": null, "blocks": "S4", "advisor": "operator", - "status": "sorted", + "status": "decided", "foldedInto": [], "class": "delegated", "reason": "Delegated test: the order of steps of an enumeration and what it records, inside what D9 and Law 11 rule; no Spec under `design/specs` names where the application's tenant list comes from, which the lean names as the gap.", @@ -6152,11 +6152,11 @@ "changes": "No Spec: `spec:application.rebuild` at its tenant rules and `spec:application.generation-registry` at its `startGeneration` Contract bullet carry it. The register row alone. The alternative, the enumerated-tenants record, would add a field to `tableGenerationProgress` and to the restore's check scope [design/specs/application/restore.sdp.md:96] and a growth bound under F13.", "ifWrong": "A tenant with streams older than a generation and no row in the list when a pass reached its position is served no row of the new generation until `fillTenants` and a rebuild, one tenant's stale read model, found when its queries answer no rows; a count of tenants visited on the progress row, or the record the earlier lean asked for, is a field added by an edit and changes no stored row's meaning." }, - "checkedBy": null, + "checkedBy": "gpt-6.1-sol, medium, 2026-10-03: 26 of 31 claims held; corrections: CONTEXT.md:135-136 for the grant read in the same transaction, the earlier lean is not kept in the row, native L2-5 runs exist, journal.sdp.md:41 says affected rows stay one version behind until a command or verify pass rewrites them, and rebuild.sdp.md:70 lets live writes insert missing rows in verifying and verified, which narrows the downside the ifWrong names (job-mus2kbol-a1d74933, plant caught)", "fork": null, - "decidedBy": null, - "decidedOn": null, - "ruling": null + "decidedBy": "advisor:operator", + "decidedOn": "2026-10-03", + "ruling": "By the lean of the domain lens on this operator row, under the owner's one-advisor word, its facts checked with the corrections in checkedBy, none of which touches its do: the Specs as written; a tenant whose row enters the list behind a backfill's cursor is not visited by that pass and needs no visit, the verify pass starts again at the first tenant, a tenant fill and a generation in flight exclude each other, an operator runs fillTenants before the first startGeneration on a deployment whose grants predate the list, and no pass records the tenants it enumerated" }, { "id": "OD-150", From 586ecb351275685ae4a4a2cf15b33db57c2005f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:31:30 +0200 Subject: [PATCH 114/155] build: 4d an expired tombstone is swept --- src/command/receipts.ts | 21 +++++++++---------- tests/simulator/receipts-sweep.test.ts | 28 ++++++++++++++++---------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/src/command/receipts.ts b/src/command/receipts.ts index 25bbdf6..293b353 100644 --- a/src/command/receipts.ts +++ b/src/command/receipts.ts @@ -148,19 +148,16 @@ export async function sweep( const rows = await expired().take(size); let deleted = 0; for (const row of rows) { - // The command path has no tombstone retention or replay. Refuse rather than destroy one. - if (row.tombstone) - throw new Error( - "Receipt sweep cannot handle tombstones until lookup supports them", + if (!row.tombstone) { + const classified = classifyReceipt( + row, + row.fingerprint, + row.contractVersion, + now, ); - const classified = classifyReceipt( - row, - row.fingerprint, - row.contractVersion, - now, - ); - if (classified.class !== "new") - throw new Error("Receipt sweep expected an expired receipt"); + if (classified.class !== "new") + throw new Error("Receipt sweep expected an expired receipt"); + } await ctx.db.delete(row._id); deleted++; } diff --git a/tests/simulator/receipts-sweep.test.ts b/tests/simulator/receipts-sweep.test.ts index 0043576..3aeb3b4 100644 --- a/tests/simulator/receipts-sweep.test.ts +++ b/tests/simulator/receipts-sweep.test.ts @@ -118,29 +118,35 @@ for (const composition of compositions) { }), ).toEqual({ deleted: 2, compacted: 0, more: true }); }); - // The absent irreversible retention path is a gap against the Spec's expiry narrative. - test(`convex-test: ${composition.name} refuses a tombstone and rolls back all sweep deletes`, async () => { + // spec:command.receipt-table tombstoneShape, expiryDefaults and fnSweep. + test(`convex-test: ${composition.name} deletes an expired tombstone and leaves an unexpired tombstone untouched`, async () => { const t = composition.app(); - await t.run(async (ctx) => { + const kept = await t.run(async (ctx) => { await ctx.db.insert("receipts", receipt("a", "reversible", now - 1)); await ctx.db.insert("receipts", { - ...receipt("a", "irreversible"), + ...receipt("a", "expired-tombstone"), tombstone: true, affected: [], versions: [], }); + const id = await ctx.db.insert("receipts", { + ...receipt("a", "unexpired-tombstone", Number.MAX_SAFE_INTEGER), + tombstone: true, + affected: [], + versions: [], + }); + return ctx.db.get(id); }); - const before = await t.run((ctx) => ctx.db.query("receipts").collect()); - await expect( - t.mutation(composition.internal.receipts.sweep, { + expect( + await t.mutation(composition.internal.receipts.sweep, { tenantId: "a", now, limit: 1000, }), - ).rejects.toThrow("until lookup supports them"); - expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual( - before, - ); + ).toEqual({ deleted: 2, compacted: 0, more: false }); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual([ + kept, + ]); }); // spec:command.actor-and-scope fnNextTenant and the receipt sweep's per-tenant boundary. test(`convex-test: ${composition.name} sweep loop visits one tenant per run and finishes the current tenant first`, async () => { From f3723119fb869525929145516c1cdad651346de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:31:54 +0200 Subject: [PATCH 115/155] design: hosted the Spec patch with sitting two's words Applies patch-hosted-specs.diff from the tooling unit without its register hunks, which are the main thread's: E-59, the four hosted examples, the local example of the undeclared table, and the hosted rules and Design keys of the native harness. Its words follow sitting two: a planting record, a test result, the four tier names with the target recorded apart, the usage reading, the stated plan and the backup archive. The selector accepts the regional deployment URL the CLI prints. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 7 ++-- ...lpers.probe-3-hosted-function-calls.sdp.md | 31 ++++++++++++++ ...ions.probe-7-hosted-in-place-import.sdp.md | 34 ++++++++++++++++ ...exclude-pending-scheduled-functions.sdp.md | 2 +- ...ailed-runs.probe-7-hosted-retention.sdp.md | 31 ++++++++++++++ ...d-functions-table-shows-failed-runs.sdp.md | 2 +- design/specs/facts/probe-plan.sdp.md | 3 +- .../specs/platform/acceptance-contract.sdp.md | 3 +- ...ment-accepts-production-composition.sdp.md | 32 +++++++++++++++ design/specs/platform/native-harness.sdp.md | 40 ++++++++++++++----- ...survives-another-composition-deploy.sdp.md | 29 ++++++++++++++ 11 files changed, 198 insertions(+), 16 deletions(-) create mode 100644 design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls.sdp.md create mode 100644 design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.sdp.md create mode 100644 design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.sdp.md create mode 100644 design/specs/platform/native-harness.hosted-deployment-accepts-production-composition.sdp.md create mode 100644 design/specs/platform/native-harness.undeclared-table-survives-another-composition-deploy.sdp.md diff --git a/design/README.md b/design/README.md index 7659de1..7a31223 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -232 specs · 5 packs · 131 anchors → 368 nodes · 1225 edges (0 errors, 0 warnings) +237 specs · 5 packs · 131 anchors → 373 nodes · 1235 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` @@ -122,7 +122,7 @@ The doc numbers its decisions, laws, sources and probes. The plan numbers everyt | `S1` to `S15` | source | | `Sc L0-1` to `Sc L6-4`, `Sc ALL-1` | acceptance scenario, by layer then table order | | `OQ1` to `OQ6` | open question of the doc, in bullet order | -| `E-1` to `E-58` | design extension beyond the doc, register below | +| `E-1` to `E-59` | design extension beyond the doc, register below | | `Vocab: term` | a term of the vocabulary Spec | | `Thesis`, `Decision method rule n`, `Fact ledger`, `Probes`, `Acceptance scenarios`, `First experiment`, `Existing systems`, `Vocabulary`, `Sources` | an unnumbered section of the doc, cited by name | @@ -180,7 +180,7 @@ Every row of the doc's acceptance table is an example Spec that `refines` and `v ## The extension register -An extension is a design claim the doc does not make. Each one is three things together: a bullet that starts with `[extension]` and cites its E-number, and a `[non-blocking] Extension E-n` open question in the Intent of the Spec that owns it. The register lists every E-number once. "Owned by" names the Specs that carry the open question and the ruling; other Specs cite the number as a cross-reference without repeating the question. Every extension is a provisional ruling the owner confirms, changes or rejects; none is blocking, because each has a working default. E-18, E-19, E-27 to E-29 and E-59 are unused; E-13 and E-39 were assigned by the integration round from the unassigned and the Package C range, E-14 by review round 2, and E-15 and E-16 by slice S0. +An extension is a design claim the doc does not make. Each one is three things together: a bullet that starts with `[extension]` and cites its E-number, and a `[non-blocking] Extension E-n` open question in the Intent of the Spec that owns it. The register lists every E-number once. "Owned by" names the Specs that carry the open question and the ruling; other Specs cite the number as a cross-reference without repeating the question. Every extension is a provisional ruling the owner confirms, changes or rejects; none is blocking, because each has a working default. E-18, E-19 and E-27 to E-29 are unused; E-13 and E-39 were assigned by the integration round from the unassigned and the Package C range, E-14 by review round 2, and E-15 and E-16 by slice S0. | E | Owned by | What the doc leaves open, and the option taken | |---|---|---| @@ -237,6 +237,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-56 | [`obligations.retention-and-restore`](specs/obligations/retention-and-restore.sdp.md) | External dispatch switched by the `OBLIGATIONS_DISPATCH` environment variable, read only in the parent; the cursor-driven `rebuildSchedules` operation, which refuses while `MAINTENANCE_MODE` is `restore`; the switch order, dispatch off before the snapshot and on last, after the transactional restore has reopened writers | | E-57 | [`obligations.record-contract`](specs/obligations/record-contract.sdp.md), [`obligations.sweeper`](specs/obligations/sweeper.sdp.md) | Three deployment-wide indexes lead with `status`, not `tenantId`; a recorded deviation from the review rubric's item 25 | | E-58 | [`effects.claim-call-settle`](specs/effects/claim-call-settle.sdp.md), [`effects.external-effects`](specs/effects/external-effects.sdp.md) | The lease is the runtime's action timeout plus 5 minutes; the provider key is `tenantId:effectKey`; ten late reports kept; a reconcile returns the same report union; plus three bounded repeats of a thrown settle call with a duplicate rule in the settle, so a repeat is not an attempt under Law 12 | +| E-59 | [`platform.native-harness`](specs/platform/native-harness.sdp.md) | A native run on a hosted deployment: the same harness, run record and compositions on one hosted development deployment in a team and project of their own, selected only by `npm run test:hosted` from process variables, one run at a time, from a workflow whose failure fails no other check; the deploy key reaches a CLI child only as `CONVEX_DEPLOY_KEY` in its environment, and the admin access removes every form of it where a response is first seen; the record names the deployment as the backend it ran on, claims no parity under Sc L2-9 and passes no scenario; the hosted halves of Probe 3 and Probe 7, the production composition's acceptance on the deployment, and one local example that a table's documents survive another composition's deploy, are its examples | ## Open questions for the owner diff --git a/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls.sdp.md b/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls.sdp.md new file mode 100644 index 0000000..55dade3 --- /dev/null +++ b/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls.sdp.md @@ -0,0 +1,31 @@ +--- +id: spec:facts.f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f04-nested-calls-cost-more-than-helpers + verifies: spec:facts.f04-nested-calls-cost-more-than-helpers +--- +# Probe 3: the function calls a component call costs on a hosted deployment + +Probe 3 · native backend tier · temporary copy of the fixture composition · hosted deployment. + +## Intent + +- outcome: The function calls that one parent mutation through component queries adds to a hosted deployment's usage, against the same mutation through a helper, are read from the deployment's usage readings and recorded. (Probe 3, F4, E-59) + +```gwt +Given the hosted deployment running the hosted driver's temporary copy of the fixture composition, and a parent mutation that reads the same document {reads: 50} times through a helper function or through a component query +When a client calls the parent mutation {calls: 20} times through the component query, then {calls: 20} times through the helper, then {calls: 20} times through the component query again, and the run reads the deployment's function calls before and after each set of calls +Then the function calls the helper set adds are {helperCallsPerParentCall: 1} per parent call +And the function calls each component set adds are recorded, and the count is settled only when the two component sets add the same number +``` + +## Verification — executable + +- Runs last in the hosted driver, on the copy the retention example ran on, which stays deployed for the next run; it starts no local backend. +- The function calls are the `functionCalls` metric of `GET /api/v1/get_current_usage`, read with the deploy key; after a set the run reads again until two readings in a row agree, and the run's evidence holds every usage reading with its time and its `seedStatus`. +- The helper set is the control: it must add exactly 20 function calls, one per parent call. The count is settled only when the control held, the two component sets added the same number, every `seedStatus` was complete and no UTC day boundary fell between the first and the last reading; otherwise the record says the count is not settled and the test result is failed, because the run did not reach its boundary. +- The endpoint is beta, may understate usage and promises no time by which a call is counted, so a settled count is the strongest reading this design can make, not a provider's promise; it is one hosted deployment's count on one plan and date, and it is the run's own only when no other run on the deployment overlapped it. +- No admin read and no other call runs between the two readings of a set. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.sdp.md new file mode 100644 index 0000000..325a7d9 --- /dev/null +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.sdp.md @@ -0,0 +1,34 @@ +--- +id: spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f12-backups-exclude-pending-scheduled-functions + verifies: spec:facts.f12-backups-exclude-pending-scheduled-functions +--- +# Probe 7: replacement in place on a hosted deployment preserves schedules that run against restored data + +Probe 7 · native backend tier · temporary copy of the fixture composition · hosted deployment. + +## Intent + +- outcome: CLI export and replacement import in place on a hosted deployment restore the exported documents and keep the destination's scheduler rows, as the local backend does. (Probe 7, F12, E-59) + +```gwt +Given the hosted deployment running the hosted driver's temporary copy of the fixture composition, which adds the scheduler references and reactions of the local in-place example +When it exports business documents and five scheduler states with the deploy key, adds documents and schedules, then imports the backup archive in place before pending reactions are due +Then documents including ids and creation times in {scopes: "parent and every mounted component"} equal the exported documents {equal: true} +And every scheduler row in each scope is unchanged {unchanged: true} +And later business changes are gone {removed: true} and the destination environment is unchanged {environmentUnchanged: true} +And kept reactions read {value: "exported"} no earlier than their scheduled time {onTime: true} +And each restored scheduler reference resolves to the kept row {found: true} and cancellation leaves state {state: "canceled"} +``` + +## Verification — executable + +- Runs second in the hosted driver, after F16's retention example; it starts no local backend. +- The copy is the one the retention example ran on, so the table `plantedSchedules` stays in the export and the import restores it as it was at the export. +- The bound values are what the local backend showed on F12; a hosted answer that differs is a finding on F12. +- The test changes business data before the import and fails if it does not reach that boundary; every kept pending schedule is let run before the test ends, so that no later example finds it pending. +- Hosted dashboard restore is not run: the deploy key reaches the deployment through the CLI's export and import only. The fresh-destination case needs a second deployment and is not run. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md index 8645ede..88fe113 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md @@ -18,7 +18,7 @@ The backup page excludes scheduled functions, code, configuration and environmen ### Open questions -- [non-blocking] Hosted dashboard restore, file storage, Workpool and Workflow are not run; local CLI import does not settle those paths. (F12, Probe 7) +- [non-blocking] Hosted dashboard restore, file storage, Workpool and Workflow are not run; local CLI import does not settle those paths. CLI replacement in place on a hosted deployment is the hosted example `spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import`; dashboard restore has no example. (F12, Probe 7, E-59) ## Constraints diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.sdp.md new file mode 100644 index 0000000..02db1f9 --- /dev/null +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.sdp.md @@ -0,0 +1,31 @@ +--- +id: spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention +kind: example +altitude: story +readiness: defined +relations: + refines: spec:facts.f16-scheduled-functions-table-shows-failed-runs + verifies: spec:facts.f16-scheduled-functions-table-shows-failed-runs +--- +# Probe 7: completed schedules on a hosted deployment are read at the ages runs reach + +Probe 7 · native backend tier · temporary copy of the fixture composition · hosted deployment. + +## Intent + +- outcome: Terminal schedules that earlier runs planted are read or found missing at their recorded age, so that the records of runs at different ages bracket the window the deployment keeps them for. (Probe 7, F16, E-59) + +```gwt +Given the hosted deployment running the hosted driver's temporary copy of the fixture composition, which keeps the parent's schedules earlier runs planted and, in the table `plantedSchedules`, a planting record for each that names the run that planted it +When a run reads `plantedSchedules` and the parent's `_scheduled_functions` before it deploys or imports anything, then plants one succeeded, one failed and one canceled schedule +Then each planting record is recorded with its age and whether its schedule was read {recorded: true} +And the run's own schedules are read in states {states: "success,failed,canceled"} each with a `completedTime` {completedTime: true} +``` + +## Verification — executable + +- Runs first in the hosted driver, before any deploy or import; it starts no local backend. When the deployment holds no planting record, the driver deploys the copy first and the run records that it read nothing. +- A planted schedule's age is the reading's time less the `completedTime` its planting record holds, so it is known after the scheduler row is gone; the ages are read, never assumed from the schedule, because GitHub may delay or drop a scheduled run. +- No age is bound: the ages are measured times, recorded in the run's evidence and on the fact and not asserted. The run reaches its boundary when it has planted its three schedules and recorded every earlier planting record, and its test result is failed otherwise; it is never skipped. A run's test result can be passed without showing a retention window, and the window stays open on F16 until records show it. +- The records show the window when one shows a schedule present at one age and a later one shows it missing; with runs on Monday and Thursday the nominal ages are 3, 4, 7, 10, 11 and 14 days, and each recorded age uses the actual completion and reading times, so a bracket is no finer than those steps. +- The run records whether a backup archive was imported on the deployment since a schedule was planted. Only the parent's schedules are planted, because a later deploy of another composition may unmount a component. diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md index 788b2a3..7c2ee0f 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.sdp.md @@ -18,7 +18,7 @@ The scheduled-functions page documents five states and a seven-day completion-re ### Open questions -- [non-blocking] Seven-day retention and expiry, hosted dashboard restore, Workpool and Workflow are not run; the short native retention observation does not establish the documented window. (F16, Probe 7, F12) +- [non-blocking] Seven-day retention and expiry, hosted dashboard restore, Workpool and Workflow are not run; the short native retention observation does not establish the documented window. The hosted example `spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention` records the ages at which planted schedules are read or missing, so that runs bracket the window a hosted deployment keeps. (F16, Probe 7, F12, E-59) ## Constraints diff --git a/design/specs/facts/probe-plan.sdp.md b/design/specs/facts/probe-plan.sdp.md index 380ac90..b7150f1 100644 --- a/design/specs/facts/probe-plan.sdp.md +++ b/design/specs/facts/probe-plan.sdp.md @@ -41,7 +41,7 @@ The probes are not the first experiment. The experiment builds Layers 0 to 2 and ### Open questions - [non-blocking] Probe 1 ran on 2026-10-01: a closed client leaves its mutation executed once or not at all, a backend restart leaves it executed once, and the HTTP client does not retry; whether callers of `ConvexHttpClient` must supply a request key is D6's question for the owner (Probe 1, D6) -- [non-blocking] Probe 3 ran on 2026-10-01 for latency on a local backend; a local backend has no function-call quota, so that half waits for a hosted deployment (Probe 3, F4, D2, D8) +- [non-blocking] Probe 3 ran on 2026-10-01 for latency on a local backend; a local backend has no function-call quota, so that half is the hosted example `spec:facts.f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls`, which only a hosted deployment can run (Probe 3, F4, D2, D8, E-59) - [non-blocking] Probe 5 ran on 2026-10-01: a parent query over a component query stays reactive, pages pinned by their end cursors stay contiguous, and a capped page crosses the boundary as `SplitRequired`; the `convex-helpers/react` hook then lost rows, and slice S2 decides how a context list is capped before it builds one (Probe 5, F15, S4, D8) - [non-blocking] Probe 6 records native backend evidence on the pins named by F17: a row race, cancel and resume, thrown-batch rollback, restart, table scope and a parent batch returning a context cursor to the migrations driver; hosted quotas and sustained contention are not run. (Probe 6, F17, D9) - [non-blocking] Probe 7 observes all five scheduler states, local CLI replacement into a fresh backend and in place, stored scheduler IDs, kept reactions, scheduled-argument limits and failed-function scan cost; seven-day expiry, hosted dashboard restore, Workpool and Workflow remain open before the decisions resting on those observations. (Probe 7, F12, F13, F16, D13, D19) @@ -81,6 +81,7 @@ Each probe is a fixture-app test on a disposable backend. The probe app is separ - evidence: each run records commit, backend and dependency versions, configuration, command and result, and states any difference from production configuration (Acceptance scenarios) - probe7Narrowed: native scheduler and CLI import cases deploy temporary copies of the production composition from modules under `tests/native/`, with added functions and schema hashes recorded; no probe function ships in the example or fixture; local observations do not establish hosted restore or seven-day expiry (F12, F13, F16, S6, S8) - location: the probe app lives in this repository, beside the design, as the owner ruled under OQ4 on 2026-10-01 (OQ4) +- hostedHalves: [extension] Probe 3's function-call quota, Probe 7's retention window and Probe 7's replacement in place each have an example that runs as a native test on the one hosted deployment of `spec:platform.native-harness`, on F4, F16 and F12, the retention window recorded as ages and never bound at seven days; they are the exception to `backend`, because they run on that hosted deployment and not on a disposable local backend, and their probe functions ship in no composition, deployed only in a temporary copy of the fixture composition; hosted dashboard restore has none, because the harness reaches a hosted deployment only through the CLI and the deployment's own routes (E-59, F4, F12, F16) ## Verification — reviewed diff --git a/design/specs/platform/acceptance-contract.sdp.md b/design/specs/platform/acceptance-contract.sdp.md index 5d40229..3d84009 100644 --- a/design/specs/platform/acceptance-contract.sdp.md +++ b/design/specs/platform/acceptance-contract.sdp.md @@ -49,6 +49,7 @@ The doc's acceptance table is the contract. It lists forty-three scenario rows b - [extension] The acceptance check takes the rows a Spec requires and one run's record, derives its required scenarios from the graph, and passes only when every required scenario passed (E-17, First experiment) - [extension] A required scenario is missing when its row has no example or its example has no enabled verifier, absent when the record holds no result of a verifier it has, failed when a result of a verifier is not passed or came from another tier or another composition than the example names, and passed otherwise; when its verifiers differ, failed wins over absent, so a scenario with one verifier failed and another absent is failed (E-17, Acceptance scenarios) - [extension] The acceptance check passes only on a record whose run passed and whose tree was clean, and its answer names the record's commit (E-17, Acceptance scenarios) +- [extension] A native run on a hosted deployment is evidence about that deployment, its plan and its date, and passes no scenario: the acceptance check reads only a record whose `target` is absent or `local backend` (E-59, E-17, Sc L2-9) - [extension] The corpus keeps no list of the scenarios that have a verifier or that passed: the first is read from the graph and the second from a record, each time the check runs (E-17) ## Design @@ -67,7 +68,7 @@ The corpus maps the doc's readiness words onto SDP's rungs without collapsing th - typeAcceptanceVerdict: [extension] `interface AcceptanceVerdict { result: "passed" | "not passed"; commit: string; clean: boolean; runResult: "passed" | "failed" | "interrupted"; scenarios: (RequiredScenario & { status: "passed" | "failed" | "absent" | "missing"; reason: string | null })[] }` (E-17) - acceptanceRecord: [extension] the record the check reads is the run record of `spec:platform.native-harness`, made by one run of every test project together, so that it holds the results of the domain and build tiers beside the native ones (E-17, E-15) - acceptanceJoin: [extension] a verifier's results are the record's test entries whose `file` is the verifier's file; a scenario passed when each of its verifiers has at least one result, every result is `passed` and of the project that runs the scenario's tier, `pure` for domain, `simulator` for simulator, `native` for native and end to end and `types` for build, and, for a native or end-to-end scenario, the example names a composition, the results between them name at least one backend, and every backend they name has that composition (E-17, E-15, E-14) -- acceptanceCommand: [extension] `npm run acceptance` checks the rows `acceptanceRows` of `spec:application.first-experiment` names against the newest record under `evidence/runs/` or against the record a path argument names; a candidate for the newest record is a `.json` file directly under `evidence/runs/` that parses as an object with a text `commit`, a boolean `clean`, a `result` of `passed`, `failed` or `interrupted`, a list `tests` and a text `startedAt`, and the newest is the candidate whose `startedAt` is the largest, with equal `startedAt` values resolved by choosing the file whose name sorts last; a record that cannot be parsed, that lacks those fields other than `startedAt`, or that has a test entry that is not an object with a text `name`, a text `file`, a `result` of `passed`, `failed` or `skipped`, a `project` that is absent or text and a list `backends` of objects whose `composition` is absent, `null` or text, is read as no record, and no older record is tried; it prints one line per required scenario with its row, example, tier, composition, verifier files, status and reason, and then one line, `acceptance: passed` or `acceptance: not passed`, with the count of each status, the run's result, whether the tree was clean and the record's commit (E-17) +- acceptanceCommand: [extension] `npm run acceptance` checks the rows `acceptanceRows` of `spec:application.first-experiment` names against the newest record under `evidence/runs/` or against the record a path argument names; a candidate for the newest record is a `.json` file directly under `evidence/runs/` that parses as an object with a text `commit`, a boolean `clean`, a `result` of `passed`, `failed` or `interrupted`, a list `tests`, a text `startedAt` and a `target` that is absent or `local backend`, and the newest is the candidate whose `startedAt` is the largest, with equal `startedAt` values resolved by choosing the file whose name sorts last; a record that cannot be parsed, that lacks those fields other than `startedAt`, whose `target` is present and not `local backend`, or that has a test entry that is not an object with a text `name`, a text `file`, a `result` of `passed`, `failed` or `skipped`, a `project` that is absent or text and a list `backends` of objects whose `composition` is absent, `null` or text, is read as no record, and no older record is tried; it prints one line per required scenario with its row, example, tier, composition, verifier files, status and reason, and then one line, `acceptance: passed` or `acceptance: not passed`, with the count of each status, the run's result, whether the tree was clean and the record's commit (E-17) - acceptanceExit: [extension] the command exits 0 when the check passes; 1 when a required scenario is failed or absent, the record's run did not pass or its tree was not clean; 3 when none of those holds and a required scenario is missing; and 2 when it cannot answer, which is no record, a graph that does not derive, a Spec that names no required row, an example Spec in the graph whose opening line begins with `Sc ` and cannot be read whole, including one whose row cannot be read, whether or not its row is required, or any other failure to answer; for every failure to answer it prints the one line `acceptance: cannot answer` with the reason and exits 2, so a failure to answer never ends without that line (E-17) ## Verification — reviewed diff --git a/design/specs/platform/native-harness.hosted-deployment-accepts-production-composition.sdp.md b/design/specs/platform/native-harness.hosted-deployment-accepts-production-composition.sdp.md new file mode 100644 index 0000000..3189216 --- /dev/null +++ b/design/specs/platform/native-harness.hosted-deployment-accepts-production-composition.sdp.md @@ -0,0 +1,32 @@ +--- +id: spec:platform.native-harness.hosted-deployment-accepts-production-composition +kind: example +altitude: story +readiness: defined +relations: + refines: spec:platform.native-harness + verifies: spec:platform.native-harness +--- +# A hosted deployment accepts the production composition and answers an ordinary client + +E-59 · native backend tier · production composition · hosted deployment. + +## Intent + +- outcome: The production composition deploys to the hosted deployment with the deploy key, the fixture issuer's key set is taken from the deployment's environment variables, and an ordinary client carrying a fixture token is answered, which no run on the local backend shows. (E-59, E-13) + +```gwt +Given the hosted deployment named by the run's deployment URL and deploy key, with the fixture issuer's variables set on it +When the run deploys the production composition, grants a subject with admin access, and an ordinary client carrying that subject's token places an order of {lines: 1} line +Then the deploy completes {deployed: true} +And the command's outcome is {outcome: "applied"} +And the function log read with the deploy key holds the command's completion record {records: 1} +``` + +## Verification — executable + +- Runs third in the hosted driver, after F12's replacement in place and before Probe 3; it starts no local backend. +- The run's record names the deployment, the deploy key by its name, the stated plan, the CLI version, the backend version the deployment reports or `null`, the usage readings before and after, and the deploy's wall time. +- Convex documents a data URI as a `customJwt` key set; the example shows whether this composition with that key set is accepted on this deployment, on its plan and its date. +- The function log is read as the run goes and what it saw is recorded; the local backend's log of its last 1000 entries is not taken as the hosted deployment's bound, which is not published. +- A passed test result claims no parity with a release and passes no scenario row. diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 84f6bb7..6432039 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -8,9 +8,9 @@ relations: --- # The native harness -Feature · Detail: extension · Traces: Acceptance scenarios (tiers, evidence), S13, Sc L2-9, D11, E-13, E-15. +Feature · Detail: extension · Traces: Acceptance scenarios (tiers, evidence), S13, Sc L2-9, D11, E-13, E-15, E-59. -The doc says that native tests prove component and nested-mutation behavior, scheduling, contention and deployment, that the kernel's fixture app is separate from the example app, that test-only functions never ship and that every test owns its disposable backend. It does not say how a test obtains an identity, what the fixture app contains, how a fault enters a run, what a test may read or what it controls. This Spec carries those rules as two extensions, E-13 and E-15. Every rule here is the corpus's provisional reading until the owner rules, except the local reading of Sc L2-9 in the first rule, which is settled. The doc's own rules on tiers and evidence stay in `spec:platform.acceptance-contract`. +The doc says that native tests prove component and nested-mutation behavior, scheduling, contention and deployment, that the kernel's fixture app is separate from the example app, that test-only functions never ship and that every test owns its disposable backend. It does not say how a test obtains an identity, what the fixture app contains, how a fault enters a run, what a test may read or what it controls. This Spec carries those rules as two extensions, E-13 and E-15, and a third, E-59, for a native run on a hosted deployment. Every rule here is the corpus's provisional reading until the owner rules, except the local reading of Sc L2-9 in the first rule, which is settled. The doc's own rules on tiers and evidence stay in `spec:platform.acceptance-contract`. ## Intent @@ -24,6 +24,7 @@ The doc says that native tests prove component and nested-mutation behavior, sch - [non-blocking] Extension E-13: the doc says native acceptance runs with production authority and a disposable backend per test and does not say how a test obtains a signed identity, which a production issuer cannot mint for a throwaway backend; the option taken here is a fixture issuer: a composition that runs natively reads its issuer, application ID and key set from the deployment's environment variables, the harness sets the key set to a data URI whose private key it holds, and each caller under test is an ordinary client carrying a token the harness signs; the option first taken, the backend's admin key acting as an identity, was dropped because such a caller passes Convex's visibility check for internal functions, which `spec:platform.native-harness.admin-key-acting-as-identity-reaches-internal-function` shows; this reading reaches into production, because running the production composition natively with no changed file means the example application's `auth.config.ts` is a `customJwt` provider whose trust root is whatever the deployment's environment names, and a provider that fetches its keys from an issuer's own endpoint cannot be run on a disposable backend this way; the owner rules on both halves, the test identity and the production provider (E-13, S13, Sc L2-9, D11) - [non-blocking] Extension E-15: the doc does not say what the fixture app contains, how a fault enters a run, what a test may observe, or what a test controls on a backend that offers no clock and no scheduler switch; the option taken here is the rules below; two of them read the doc for the owner: the production-configuration run of Sc L2-9 is taken as the scenarios routed to the production composition, where the corpus earlier asked for the full Layer 0 to 2 suite on a release build, and evidence records that back a pass claim are kept in the repository; whether the durations Layer 3 pins become configuration, so that a test can outwait them, is left to Layer 3; and because the Convex CLI reads `convex.json` and `package.json` from its working directory, a composition is a name, a project directory and a functions directory, the production composition's project directory is `example/` with a `package.json` that links to the repository's one, and both compositions' generated files are committed; the owner has not ruled on the layout (E-15, Acceptance scenarios, Sc L2-9) - [non-blocking] Extension E-15: the design takes the fixture action `nonUiCaller:send`, which any authenticated client of the fixture composition may call to run any fixture command's internal entry under any namespace and actor kind, as the fixture's stand-in for trusted server code, kept out of the production composition; it is a provisional reading the owner rules, because within the fixture composition it lets a client do what `spec:command.tenancy-and-authority` refuses a client (E-15, D11, Sc L1-7) +- [non-blocking] Extension E-59: the doc's native tier is a test on a real backend, S13 names a local one, and Sc L2-9 is read as the local observation; the doc does not say whether a native test runs on a hosted deployment, where alone Probe 3 can read function-call quota and where F12 and F16 keep parts that only documentation states; the option taken here: the same harness, run record and compositions run natively on one hosted development deployment in a Convex team and project of their own, selected by process variables under a test configuration of its own, one run at a time, from a workflow whose failure fails no other check, with one development deploy key limited to the actions `deployKeyActions` lists; the alternatives were each developer's own deployment, which still needs one more key for the continuous integration, a preview deployment per run, which on the Free and Starter plans expires by default five days after it is created, before F16's seven-day window, and a fresh development deployment per run, which needs a project key, a credential the docs give total control over a project; the team, the project, the plan and the key are the owner's to provide (E-59, Sc L2-9, F4, F12, F16, Probe 3, Probe 7) ## Rule @@ -33,8 +34,17 @@ The doc says that native tests prove component and nested-mutation behavior, sch - [extension] No caller under test carries the admin key, alone or acting as an identity, because a call made with it passes Convex's visibility check (E-13) - [extension] The fixture issuer is the identity source of a native run and its one named difference from production; it is recorded as the identity source and is not an adjusted configuration (E-13, Sc L2-9) - [extension] The harness uses the admin key to deploy, to generate code, to set and read environment variables, to read stored data and the function log, to run fixture functions, and to call a composition's internal functions that admin access reaches, its grant mutation and its operator entries (E-15, E-48) -- [extension] Every Convex CLI call the harness makes names the backend it started on the command line and gets an environment of its own, so that no variable of the caller's shell and no dotenv file selects another deployment (E-15) +- [extension] On the local backend, every Convex CLI call the harness makes names the backend it started on the command line and gets an environment of its own, so that no variable of the caller's shell and no dotenv file selects another deployment (E-15) - [extension] No generated secret, the instance secret or the admin key, reaches an error message, test output or a record (E-15) +- [extension] A native run on a hosted deployment uses the same harness, run record and compositions as one on the local backend and names the hosted deployment as the backend it ran on; it claims no parity with a release under Sc L2-9, passes no scenario row, and shows only what the local backend cannot: the deploy key's authority, the deployment's acceptance of a composition, its reachability, its report of its own usage and what it keeps between runs (E-59, Sc L2-9) +- [extension] Only `npm run test:hosted` selects a hosted deployment, by the deployment URL and the deploy key in two process variables; no other variable, no dotenv file and no login selects it, `npm run test:all` cannot select it, and a run on the local backend reads neither variable (E-59) +- [extension] The harness refuses a native run on a hosted deployment whose deploy key does not begin `dev:`, whose deployment URL is not `https://`, the deployment name the key carries and `.convex.cloud`, with or without one region between the name and `.convex.cloud` as the CLI prints it (`https://..convex.cloud`), or whose composition's project directory holds a `.env` or `.env.local` file, so a developer's own `.env.local` at the repository root refuses it; a refusal names the variable or the file and never a value (E-59) +- [extension] A native run on a hosted deployment starts no local backend: `fixtureBackend` and `productionBackend` refuse inside it (E-59) +- [extension] The deploy key enters the harness as one process variable and reaches a Convex CLI child only as `CONVEX_DEPLOY_KEY` in the environment the harness gives that child, never as an argument; the harness's HTTP and socket clients carry it in a header or an authentication message, never in a URL (E-59, E-15) +- [extension] No form of the deploy key, the whole key, its part after the `|` or either of them URL-encoded, reaches an error message, test output, a record or an artifact of the continuous integration; every member of the admin access to a hosted deployment and every CLI child replaces those forms in what it throws before it throws, where the response is first seen, and the record is redacted again as a whole before it is written (E-59, E-15) +- [extension] The hosted deployment keeps its data between runs, so the record of a native run on it states a kept dataset, `kept from earlier runs`, and an example on it asserts only on what its own run wrote or on what a table of the run's temporary copy says an earlier run planted (E-59) +- [extension] Native runs on the hosted deployment run one at a time, which the harness cannot enforce across machines, so the record names whether the continuous integration or a developer ran it; a failed native run on the hosted deployment fails no check of a commit's continuous integration (E-59) +- [extension] The record of a native run on a hosted deployment holds the deployment's usage readings before and after the run, with the time of each, and no example asserts a cost from them; a difference of two readings is the run's own only when no other run on the deployment overlapped it, and is unestablished when that cannot be shown; the plan it names is the stated plan, and the backend version is what the deployment reports or `null`, never mapped to a release (E-59) - [extension] An assertion about authority, identity or visibility is made through an ordinary client (E-15, E-13) - [extension] A native scenario runs on one of two compositions, the fixture composition or the production composition, and its example names which (E-15) - [extension] The fixture composition mounts the shared libraries beside fixture contexts and fixture functions; the production composition is the example application with no test-only function registered (E-15, Acceptance scenarios) @@ -50,13 +60,13 @@ The doc says that native tests prove component and nested-mutation behavior, sch - [extension] Where a scenario asserts that something did not run, the test shows the refusal's own error, shows that nothing the thing writes is stored, and shows on the same deployment that the thing exists and runs, through a control that an ordinary client sends where a client may call the thing and that is sent with admin access otherwise; where no error tells a refusal from a failure, the fixture composition supplies a part that fails the call if it runs (E-15) - [extension] An error text, a count or a wait that an example of this harness binds is what the pinned backend release answers; another answer on a new release is a finding to rule on, and the example is rebound, not the test repaired to keep the old value (E-15) - [extension] The native tier controls a run through four things: the backend process, its clients, the deployment's environment variables and, on the fixture composition, fixture functions (E-15) -- [extension] The harness starts the backend on fresh storage, stops it, kills it, and restarts it on the same storage (E-15, S13) +- [extension] The harness starts a local backend on fresh storage, stops it, kills it, and restarts it on the same storage (E-15, S13) - [extension] The harness registers the cleanup of a backend before the start of that backend can stall; a run that ends or is interrupted kills every backend process it started and removes every backend directory (E-15) - [extension] Before the harness signals a process it reads from the process table the instance name and the storage path it gave that backend, and it signals no process that does not show both; a backend whose process it cannot inspect keeps its directory and its ownership record, and the run fails naming it (E-15) - [extension] A client under test may close, and may lose a response (E-15) - [extension] The native tier cannot move the backend's clock and cannot switch its scheduler off (E-15, S13) - [extension] A scenario that needs time to pass waits in real time or shortens a duration that its owning Spec makes configuration, and a run with a shortened duration is adjusted configuration (E-15, Acceptance scenarios) -- [extension] A native run's record names the backend that ran by the hash of its executable and, when that hash is the pinned one, by its release; it names the identity source, and the commit with whether the tree was clean (E-15, Acceptance scenarios) +- [extension] A native run's record names a local backend that ran by the hash of its executable and, when that hash is the pinned one, by its release; it names the identity source, and the commit with whether the tree was clean (E-15, Acceptance scenarios) - [extension] A run's record takes its facts from the backends that ran, and a measurement is stored on the test that recorded it (E-15, Acceptance scenarios) - [extension] A claim that a tier passed cites the commit and the run of the continuous integration on it, whose record was made on a clean tree at that commit (E-15, Acceptance scenarios) @@ -78,9 +88,21 @@ The harness is plain test support with no Convex function in it. The entries who - fnFixtureBackend: [extension] `fixtureBackend(options?: { issuer?: string; executable?: Executable; parentDirectory?: string; deploy?: boolean }): Promise` starts a backend, deploys the fixture composition unless `deploy` is `false`, and removes the backend when the test finishes, however it finishes; the executable and parent directory options let the harness's own tests exercise the same cleanup path (E-15, E-13) - fnProductionBackend: [extension] `productionBackend(options?: { issuer?: string; deploy?: boolean }): Promise` starts a backend, deploys the production composition unless `deploy` is `false`, and removes the backend when the test finishes, however it finishes (E-15, E-13) - fnMeasure: [extension] `measure(name: string, value: JsonValue): void` stores a measurement on the running test (E-15) -- typeBackendFacts: [extension] `interface BackendFacts { composition: "fixture" | "production" | null; installedLayers: string[]; executable: { release: string | null; sha256: string; source: "cache" | "CONVEX_BACKEND_BINARY" }; identitySource: { kind: "fixture issuer"; issuer: string }; environment: string[]; dataset: "empty at start" }` says `null` for the composition after `deployTemporary`, so a record never claims the plain composition for a backend running a copy; the test that deployed the copy records what it added as a measurement (E-15, E-13) -- typeNativeTestEntry: [extension] `interface NativeTestEntry { project: "types" | "pure" | "simulator" | "native"; name: string; file: string; result: "passed" | "failed" | "skipped"; durationMs: number | null; errors: string[]; backends: BackendFacts[]; measurements: { name: string; value: JsonValue }[] }`; a run that includes the native project records every test it ran, of every project, each under the project that ran it (E-15) -- typeNativeRunRecord: [extension] `interface NativeRunRecord { tier: "native"; commit: string; clean: boolean; command: string; startedAt: string; finishedAt: string; result: "passed" | "failed" | "interrupted"; versions: Record; tests: NativeTestEntry[]; unhandledErrors: string[] }`; it holds no field for adjusted configuration, and a run that adjusts configuration is the trigger that adds one (E-15, Acceptance scenarios) +- typeBackendFacts: [extension] `interface BackendFacts { composition: "fixture" | "production" | null; installedLayers: string[]; target: "local backend" | "hosted deployment"; executable: { release: string | null; sha256: string; source: "cache" | "CONVEX_BACKEND_BINARY" } | null; identitySource: { kind: "fixture issuer"; issuer: string }; environment: string[]; dataset: "empty at start" | "kept from earlier runs" }` says `null` for the composition after `deployTemporary`, so a record never claims the plain composition for a backend running a copy; the test that deployed the copy records what it added as a measurement; on a hosted deployment `executable` is `null` and `dataset` is kept from earlier runs, and `environment` names the variables this run set, never a value (E-15, E-13, E-59) +- typeNativeTestEntry: [extension] `interface NativeTestEntry { project: "types" | "pure" | "simulator" | "native" | "hosted"; name: string; file: string; result: "passed" | "failed" | "skipped"; durationMs: number | null; errors: string[]; backends: BackendFacts[]; measurements: { name: string; value: JsonValue }[] }`; a run that includes the native project records every test it ran, of every project, each under the project that ran it, and `hosted` is the project of a test that ran on a hosted deployment (E-15, E-59) +- typeNativeRunRecord: [extension] `interface NativeRunRecord { tier: "native"; target: "local backend" | "hosted deployment"; commit: string; clean: boolean; command: string; startedAt: string; finishedAt: string; result: "passed" | "failed" | "interrupted"; versions: Record; tests: NativeTestEntry[]; unhandledErrors: string[]; hosted: HostedRunFacts | null }`; it holds no field for adjusted configuration, and a run that adjusts configuration is the trigger that adds one; a record with no `target` reads as one of the local backend, and `target` is recorded apart from `tier`, because a hosted deployment is a target of the native backend tier and not a tier of its own (E-15, Acceptance scenarios, E-59) +- typeHostedRunFacts: [extension] `interface HostedRunFacts { deployment: string; keyName: string; statedPlan: string; ranBy: "continuous integration" | "developer"; cliVersion: string; backendVersion: string | null; usageBefore: HostedUsage | null; usageAfter: HostedUsage | null; windowCrossed: boolean; deploys: { composition: "fixture" | "production" | null; wallMs: number }[] }`; `deployment` is the name the key carries; `keyName` and `statedPlan` are what the owner states in two variables that hold no secret, `HOSTED_DEPLOY_KEY_NAME` and `HOSTED_PLAN`, because the deployment reports its team, project and type and no plan; `backendVersion` is the answer of `/instance_version` or `null`, and `executable` stays `null`, because no mapping from that answer to a release is established; `windowCrossed` is true when a UTC day boundary falls between the two readings (E-59) +- typeHostedUsage: [extension] `interface HostedUsage { readAt: string; seedStatus: string | null; response: JsonValue }` holds the deployment's answer to `GET /api/v1/get_current_usage`, its current UTC day and calendar month, as `convex deployment usage --json` reads it; the endpoint is beta, may understate usage and promises no time by which a call is counted, so a difference of two answers is recorded in the plan's units, never in money, and is the run's own only when no other run on the deployment overlapped it (E-59) +- typeHostedTarget: [extension] `interface HostedTarget { url: string; deployment: string; keyName: string; statedPlan: string }`; the deploy key is no member, because the harness's admin access holds it and a test never reads it (E-59) +- fnHostedTarget: [extension] `hostedTarget(variables: Readonly>): HostedTarget` reads `HOSTED_DEPLOYMENT_URL`, `HOSTED_DEPLOY_KEY`, `HOSTED_DEPLOY_KEY_NAME` and `HOSTED_PLAN` and refuses as the rules say (E-59) +- fnHostedBackend: [extension] `hostedBackend(options?: { composition?: "fixture" | "production"; issuer?: string; deploy?: boolean }): Promise` deploys the composition unless `deploy` is `false`, sets the fixture issuer's variables, records the deploy's wall time and, when the test finishes, closes its clients; `stop`, `kill` and `restart` refuse, `dispose` removes nothing on the deployment, and reading `adminKey` throws (E-59, E-13) +- hostedSelection: [extension] a CLI call of a native run on a hosted deployment runs in the composition's project directory with the environment `PATH`, `HOME` set to a fresh directory, `TMPDIR` and `CONVEX_DEPLOY_KEY`, and no `--url` or `--admin-key`; `convex` 1.46.0 resolves a development deployment's deploy key to its URL and uses the key itself as the deployment's credential, in `node_modules/convex/src/cli/lib/deploymentSelection.ts` lines 714 to 749, and sends it as `Authorization: Convex` in `node_modules/convex/src/cli/lib/utils/utils.ts` lines 1373 to 1389; the local path keeps its arguments (E-59) +- deployKeyActions: [extension] the deploy key allows exactly `deployment:deploy`, `deployment:env:view`, `deployment:env:write`, `deployment:data:view`, `deployment:data:write`, `deployment:functions:runInternalQueries`, `deployment:functions:runInternalMutations`, `deployment:functions:runInternalActions`, `deployment:logs:view`, `deployment:usage:view`, `deployment:backups:view`, `deployment:backups:create`, `deployment:backups:download` and `deployment:backups:import`, the actions the routes of `AdminAccess` and the CLI's export and import check in the upstream backend at `5c7cb5b`, and not `deployment:functions:actAsUser`, pause, usage limits, integrations or the audit log (E-59) +- hostedRedaction: [extension] the secrets of a native run on a hosted deployment are the four forms of the deploy key, given to every child, every buffer and the reporter; the admin access built over a hosted deployment catches every rejection of its members, the HTTP client's raw response text and its `Invalid response` text among them, a fetch that rejects and a call that aborts, and rethrows it with those forms replaced; its HTTP and socket clients take a logger that replaces them; a CLI child's output is redacted on success as on failure and on an abort; the reporter redacts the serialized record before it writes it; the import upload token, the export download token, application data and the fixture issuer's private key are not among these secrets and are named as not covered, not scrubbed as if they were (E-59, E-15) +- hostedConfiguration: [extension] `npm run test:hosted` runs `vitest run --config vitest.hosted.config.ts`, whose one project, `hosted`, runs one driver file under `tests/hosted/` with its own global setup and the evidence reporter; the driver runs four examples in order: Probe 7's retention example before any deploy or import, Probe 7's replacement in place, the production composition's acceptance, and Probe 3's quota half; the first, second and fourth run on one temporary copy of the fixture composition that adds the modules under `tests/hosted/`, and the fourth leaves that copy deployed for the next run; when the deployment holds no earlier planting, the first run deploys the copy before the retention example and records that it read nothing; `npm run test:all` never includes it (E-59) +- hostedSchedule: [extension] the workflow `.github/workflows/hosted.yml` runs on `workflow_dispatch` and on a schedule at 03:00 UTC on Monday and Thursday; GitHub runs a schedule from the default branch, and the job runs in a GitHub environment named `hosted` that holds the key and admits the default branch only, so a dispatch from another branch gets no key; in one concurrency group that cancels no running job; it uploads the record as the artifact `hosted-evidence` only after `scripts/hosted-record-check.mjs`, reading the key from its own environment, finds no form of it in the record or the captured log, and no other workflow waits on it; until the workflow is on the default branch, the owner's machine runs `npm run test:hosted` with the variables in its shell, and its record names a developer as the one who ran it (E-59) +- hostedRedactionTest: [extension] one pure test with no network plants a key in the form of a development deploy key, gives the hosted admin access an injected fetch and a stub socket and runs a stub CLI script, each of which echoes the key in all four forms on every path, calls every member so that each fails, captures `process.stdout` and `process.stderr` and the reporter's output, writes a record into a temporary directory, and fails when any form appears in any of them (E-59) +- hostedProbeCopy: [extension] the temporary copy of the fixture composition that the hosted driver deploys adds, from modules under `tests/hosted/`, the table `plantedSchedules` of planting records, one for each schedule a run planted in the parent, with its scheduler ID, the state and `completedTime` the run saw and the `startedAt` of the run that planted it, the mutation that plants one succeeded, one failed and one canceled schedule and writes their rows, the scheduler references and reactions of the local in-place example, and Probe 3's parent mutations; the fixture composition itself gains nothing, as `probe7Narrowed` of `spec:facts.probe-plan` requires, and a later run knows a schedule's age after its scheduler row is gone (E-59) - compositionLayout: [extension] a composition is a name, a project directory and a functions directory; the Convex CLI reads `convex.json` and `package.json` from its working directory and takes no functions directory any other way, so the harness runs every CLI call in the composition's project directory; the fixture composition's project directory is the repository root, whose `convex.json` names `fixture/convex/`, and the production composition's is `example/`, whose `convex.json` names `convex/` and whose `package.json` is a symbolic link to the root's, so the repository has one `package.json` (E-15) - typeComposition: [extension] `interface Composition { name: "fixture" | "production"; project: string; functions: string; installedLayers: readonly string[] }` (E-15) - compositionTooling: [extension] code generation, the dev watch and the harness take a composition, and the type check, lint, format and the tests cover both; the generated files of both compositions are committed, the codegen script runs once per composition against a disposable backend, and CI fails when a committed generated file differs (E-15) @@ -99,7 +121,7 @@ The harness is plain test support with no Convex function in it. The entries who - observedUsageStats: [extension] a top-level mutation's completion record carries `usageStats` with `databaseReadDocuments`, `databaseWriteDocuments`, `databaseReadBytes`, `databaseWriteBytes` and `databaseWriteIndexRows`, and they count what a component call inside the mutation read and wrote (E-15) - observedFailedMutationWrites: [extension] a failed top-level mutation's completion record shows no written documents, `usageStats.databaseWriteDocuments` 0, even when a component call inside it wrote before the throw, so the function log cannot show writes that rolled back (E-15) - observedTableRead: [extension] the backend answers a read of a missing table with an empty page; the harness reads through the system query that `convex data` uses and checks an empty result against the table list (E-15) -- evidenceKept: [extension] every native run writes its record under `evidence/runs/`, which git ignores, and the continuous integration keeps the record of every push as an artifact of its run; no record is copied into the repository, and a claim names the commit and the run whose artifact holds the record (E-15, Acceptance scenarios) +- evidenceKept: [extension] every native run writes its record under `evidence/runs/`, which git ignores, and the continuous integration keeps the record of every push as an artifact of its run; no record is copied into the repository, and a claim names the commit and the run whose artifact holds the record; a native run on a hosted deployment writes its record as `hosted-` and its start time under the same directory, and its artifact is public because the repository is (E-15, Acceptance scenarios, E-59) ## Verification — reviewed diff --git a/design/specs/platform/native-harness.undeclared-table-survives-another-composition-deploy.sdp.md b/design/specs/platform/native-harness.undeclared-table-survives-another-composition-deploy.sdp.md new file mode 100644 index 0000000..8c19e07 --- /dev/null +++ b/design/specs/platform/native-harness.undeclared-table-survives-another-composition-deploy.sdp.md @@ -0,0 +1,29 @@ +--- +id: spec:platform.native-harness.undeclared-table-survives-another-composition-deploy +kind: example +altitude: story +readiness: defined +relations: + refines: spec:platform.native-harness + verifies: spec:platform.native-harness +--- +# A table's documents survive a deploy whose schema leaves the table out + +E-59 · native backend tier · temporary copy of the fixture composition, then the production composition · local backend. + +## Intent + +- outcome: The documents of a table that one deployed schema declares and the next deployed schema leaves out are read again after the first schema is deployed back, which the hosted driver relies on between one run's planting and the next run's read. (E-59) + +```gwt +Given a local backend running a temporary copy of the fixture composition whose schema declares the table `plantedSchedules`, holding {rows: 3} planting records +When the harness deploys the production composition, whose schema leaves that table out, and then deploys the same temporary copy again +Then the production composition's deploy completes {deployed: true} +And the table holds the same planting records {rows: 3} with their IDs and creation times {unchanged: true} +``` + +## Verification — executable + +- Runs at the native backend tier on the pinned local backend; it is the one local run that settles whether the hosted driver's order keeps what a run planted. +- The bound values are the expectation; a test result that shows another answer is a finding, and the hosted driver then deploys a temporary copy of the production composition that declares the table, recorded as its one difference. +- What a hosted deployment does with an undeclared table is not shown by the local backend; a run on the hosted deployment shows it. From f179f6b3d312e9d972cbe84ab0c356e15501df39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:40:09 +0200 Subject: [PATCH 116/155] build: 4f the language in the code, entry and the tier names A refusal carries the entry that refused in `entry` where it carried `commandType`: RejectionData, the wire validator, normalizeThrown, reject's callers in the pipeline, the public entry and authorizeQuery, the fixture's invalid rejection, and every test and example Spec that asserts the field. commandType stays where it names a command. The tiers are compiled, pure test, convex-test and native backend. The acceptance parser reads the four names and refuses the old ones with "names no tier"; an end to end example is the native backend tier on the production composition. Example opening lines and their "Runs in" bullets, the acceptance contract, the README and the pure tests follow. The Vitest project names stay. Co-Authored-By: Claude Opus 5.5 --- README.md | 16 ++-- ...ion.source-changes-during-recompute.sdp.md | 4 +- ...-rebuild.backfill-races-live-writes.sdp.md | 4 +- ...duplicate-out-of-order-applied-once.sdp.md | 4 +- ...consumer.partition-failure-isolated.sdp.md | 4 +- ...ns.budget-exceeded-blocks-execution.sdp.md | 4 +- ...t-runs.changed-input-voids-approval.sdp.md | 4 +- ...runs.unauthorized-proposal-rejected.sdp.md | 4 +- ...t-runs.untrusted-input-cannot-widen.sdp.md | 4 +- ...ng.timed-out-paid-call-settles-once.sdp.md | 4 +- ...acceptance-production-configuration.sdp.md | 4 +- .../first-experiment.native-contention.sdp.md | 2 +- ...riment.order-at-stock-item-id-bound.sdp.md | 4 +- .../first-experiment.order-of-1-line.sdp.md | 4 +- .../first-experiment.order-of-10-lines.sdp.md | 4 +- ...first-experiment.order-of-max-lines.sdp.md | 4 +- ...ve-stock-item-id-past-bound-refused.sdp.md | 6 +- .../specs/application/first-experiment.sdp.md | 2 +- ...nt.stock-item-id-past-bound-refused.sdp.md | 6 +- ...ry.command-without-generation-fails.sdp.md | 4 +- ...y.first-rebuild-installs-read-model.sdp.md | 4 +- ...cel-duplicate-answered-from-receipt.sdp.md | 4 +- ...ventory-example.cancel-placed-order.sdp.md | 4 +- ...ample.cancel-second-context-rejects.sdp.md | 6 +- ...ntory-example.cancel-twice-rejected.sdp.md | 6 +- ...ample.cancel-unknown-order-rejected.sdp.md | 6 +- ...xample.cancel-without-grant-refused.sdp.md | 6 +- .../specs/application/parent-use-cases.sdp.md | 2 +- ...nt-use-cases.second-context-rejects.sdp.md | 6 +- ...ent-use-cases.second-context-throws.sdp.md | 4 +- ....committed-state-read-through-query.sdp.md | 4 +- ...read-models.committed-state-visible.sdp.md | 4 +- .../read-models.list-pages-by-cursor.sdp.md | 4 +- ...els.query-refused-before-disclosure.sdp.md | 6 +- ...-models.query-refused-without-grant.sdp.md | 6 +- ...ild.online-rebuild-interrupt-resume.sdp.md | 4 +- .../rebuild.write-pause-rebuild-abort.sdp.md | 4 +- ...build.write-pause-rebuild-interrupt.sdp.md | 4 +- ...tore.restore-representative-dataset.sdp.md | 4 +- ...e-pause.restore-door-without-deploy.sdp.md | 4 +- design/specs/application/write-pause.sdp.md | 2 +- design/specs/command/actor-and-scope.sdp.md | 2 +- ...roken-argument-contract-fails-build.sdp.md | 4 +- ...aration.renamed-handler-fails-build.sdp.md | 4 +- ...peline.failure-after-journal-append.sdp.md | 4 +- ...-pipeline.failure-after-state-write.sdp.md | 4 +- ...and-pipeline.failure-before-receipt.sdp.md | 4 +- ...eceipts.capacity-refusal-then-retry.sdp.md | 4 +- ...nd-receipts.concurrent-non-ui-calls.sdp.md | 4 +- ...nd-receipts.key-reuse-changed-input.sdp.md | 4 +- ...nd-receipts.rate-refusal-then-retry.sdp.md | 4 +- ...-receipts.retry-after-lost-response.sdp.md | 4 +- ...tency-and-receipts.ui-double-submit.sdp.md | 4 +- ....rejected-then-retried-after-change.sdp.md | 4 +- design/specs/command/outcome-boundary.sdp.md | 2 +- ...ority.client-claims-agent-namespace.sdp.md | 4 +- ...rity.client-claims-system-namespace.sdp.md | 4 +- ...-and-authority.revoked-then-retried.sdp.md | 4 +- ...-and-authority.same-key-two-tenants.sdp.md | 4 +- .../command/tenancy-and-authority.sdp.md | 2 +- ...d-authority.shared-key-two-subjects.sdp.md | 6 +- ...ontext-component.competing-commands.sdp.md | 4 +- ...xt-component.competing-unique-value.sdp.md | 4 +- ...ontext-component.invalid-transition.sdp.md | 4 +- ...xt-component.stale-version-rejected.sdp.md | 4 +- .../journal.rebuild-from-baseline.sdp.md | 4 +- .../specs/decisions/d04-four-outcomes.sdp.md | 2 +- ...effects.provider-success-reply-lost.sdp.md | 4 +- ...cts.stale-worker-after-cancellation.sdp.md | 4 +- ...-effects.stale-worker-late-evidence.sdp.md | 4 +- ...robe-3-component-call-from-mutation.sdp.md | 4 +- ...s.probe-3-component-call-from-query.sdp.md | 4 +- ...ckend-restart-with-pending-mutation.sdp.md | 4 +- ...client-closed-with-pending-mutation.sdp.md | 4 +- ...confirmed.probe-1-http-client-retry.sdp.md | 4 +- ...robe-4-component-call-shares-limits.sdp.md | 4 +- ...s.probe-4-nested-call-shares-limits.sdp.md | 4 +- ...s-have-limits.probe-4-nesting-depth.sdp.md | 4 +- ...boundary.probe-2-component-boundary.sdp.md | 4 +- ...nt-boundary.probe-2-nested-mutation.sdp.md | 4 +- ...ilt-in-paginate-throws-in-component.sdp.md | 4 +- ...e.probe-5-full-first-page-not-split.sdp.md | 4 +- ...-full-first-page-split-at-equal-cap.sdp.md | 4 +- ...probe-5-hook-splits-across-boundary.sdp.md | 4 +- ...ctive.probe-5-pages-stay-contiguous.sdp.md | 4 +- ...robe-5-parent-page-outgrows-row-cap.sdp.md | 4 +- ...tive.probe-5-split-required-relayed.sdp.md | 4 +- ...active.probe-5-subscription-updates.sdp.md | 4 +- ...tamps.probe-8-parent-and-components.sdp.md | 4 +- ...me-budgets.probe-11-component-calls.sdp.md | 4 +- ...re-time-budgets.probe-9-nested-time.sdp.md | 4 +- ...fter-reading.probe-10-oversized-row.sdp.md | 4 +- ...fter-reading.probe-10-stream-budget.sdp.md | 4 +- .../domain-kernel.evaluate-twice.sdp.md | 4 +- ...ain-kernel.incremental-then-rebuild.sdp.md | 4 +- design/specs/kernel/domain-kernel.sdp.md | 8 +- design/specs/kernel/outcome-model.sdp.md | 2 +- ...bligation-module.backlog-left-alone.sdp.md | 4 +- ...tion-module.dispatch-killed-rearmed.sdp.md | 4 +- ...dule.duplicate-worker-lost-callback.sdp.md | 4 +- ...on-module.exhausted-needs-attention.sdp.md | 4 +- ...ation-module.wrapper-failed-rearmed.sdp.md | 4 +- ...and-restore.restore-behind-provider.sdp.md | 4 +- ...-restore.retention-keeps-unresolved.sdp.md | 4 +- ...line-operations.broken-audit-aborts.sdp.md | 4 +- ...erations.broken-metrics-never-abort.sdp.md | 4 +- .../specs/platform/acceptance-contract.sdp.md | 38 ++++----- ...-identity-reaches-internal-function.sdp.md | 4 +- ...ixture-issuer-token-yields-identity.sdp.md | 4 +- ...y-client-refused-component-function.sdp.md | 4 +- ...ry-client-refused-internal-function.sdp.md | 4 +- design/specs/platform/native-harness.sdp.md | 4 +- .../processes/approvals.approval-races.sdp.md | 4 +- ...esses.deploy-with-process-in-flight.sdp.md | 4 +- ...ocesses.restart-after-external-step.sdp.md | 4 +- fixture/convex/rejectionCommands.ts | 4 +- harness/acceptance.ts | 28 +++---- src/command/authority.ts | 6 +- src/command/declaration.ts | 2 +- src/command/outcome-boundary.ts | 31 +++---- src/command/pipeline.ts | 14 ++-- tests/native/cancel-order-steps.ts | 2 +- .../command-without-grant-forbidden.test.ts | 2 +- tests/native/context-queries.test.ts | 6 +- tests/native/first-experiment-steps.ts | 6 +- ...cceptance-production-configuration.test.ts | 8 +- ...irst-experiment.order-of-max-lines.test.ts | 2 +- ...ency-and-receipts.ui-double-submit.test.ts | 6 +- tests/native/order-read-steps.ts | 6 +- ...mple.cancel-second-context-rejects.test.ts | 4 +- ...tory-example.cancel-twice-rejected.test.ts | 2 +- ...mple.cancel-unknown-order-rejected.test.ts | 2 +- ...ample.cancel-without-grant-refused.test.ts | 2 +- ...rejected-then-retried-after-change.test.ts | 2 +- ...t-use-cases.second-context-rejects.test.ts | 2 +- tests/native/production-orders.test.ts | 2 +- tests/native/read-models.test.ts | 6 +- tests/native/receipted-calls.ts | 2 +- ...and-authority.revoked-then-retried.test.ts | 2 +- tests/native/tenancy-steps.ts | 2 +- tests/pure/acceptance.test.ts | 80 +++++++++++-------- tests/pure/command.test.ts | 26 +++--- tests/pure/rejection-validation.test.ts | 70 ++++++++-------- tests/pure/stock-item-id-bound.test.ts | 4 +- tests/simulator/command.test.ts | 24 +++--- tests/simulator/context-queries.test.ts | 4 +- tests/simulator/envelope-bounds.test.ts | 8 +- tests/simulator/example.test.ts | 28 +++---- tests/simulator/read-model.test.ts | 6 +- tests/simulator/receipt-disclosure.test.ts | 6 +- tests/types/reject-platform-codes.test-d.ts | 6 +- 151 files changed, 469 insertions(+), 456 deletions(-) diff --git a/README.md b/README.md index 36fa1d0..a7c53a5 100644 --- a/README.md +++ b/README.md @@ -35,18 +35,18 @@ The code runs as two compositions, two Convex apps assembled from the shared lib Each tier has its own command and its own directory: -- Domain tier: `npm run test:pure` runs `tests/pure`. These tests need no backend and no simulator. They test the kernel, the fixture domain and the pure parts of the harness. -- Simulator tier: `npm run test:simulator` runs `tests/simulator` on `convex-test`. Every test name starts with `convex-test` and the version it ran on. A simulator pass is not native proof. A test registers the composition it runs: `tests/simulator/production.ts` registers the production composition with its two contexts, and the fixture tests register the fixture composition's components themselves. -- Native tier: `npm run test:native` runs `tests/native`. Every test owns its processes and temporary storage, stops its processes and removes its storage when it ends. A test chooses its composition by the function that starts its backend: `fixtureBackend()` deploys the fixture composition and `productionBackend()` deploys the production composition. Each backend's facts in the run's record name the composition it deployed. -- Compile check: `npm run typecheck` compiles both compositions, the harness and the tests. `npm run test:types` runs the type tests in `tests/types`. A compile check says nothing about a running backend. +- Pure test tier: `npm run test:pure` runs `tests/pure`. These tests need no backend and no `convex-test`. They test the kernel, the fixture domain and the pure parts of the harness. +- `convex-test` tier: `npm run test:simulator` runs `tests/simulator` on `convex-test`. Every test name starts with `convex-test` and the version it ran on. A `convex-test` pass is not native backend proof. A test registers the composition it runs: `tests/simulator/production.ts` registers the production composition with its two contexts, and the fixture tests register the fixture composition's components themselves. +- Native backend tier: `npm run test:native` runs `tests/native`. Every test owns its processes and temporary storage, stops its processes and removes its storage when it ends. A test chooses its composition by the function that starts its backend: `fixtureBackend()` deploys the fixture composition and `productionBackend()` deploys the production composition. Each backend's facts in the run's record name the composition it deployed. +- Compiled tier: `npm run typecheck` compiles both compositions, the harness and the tests. `npm run test:types` runs the type tests in `tests/types`. A compile check says nothing about a running backend. -The claim tiers in `AGENTS.md` map to these names: compiled means compile check, pure test means domain tier, `convex-test` means simulator tier, and native backend means native tier. +The Vitest projects keep their own names: the compiled tier's type tests run in `types`, the pure test tier in `pure`, the `convex-test` tier in `simulator` and the native backend tier in `native`. The doc's end to end tier is the native backend tier on the production composition. -`npm test` runs the type tests, the domain tier and the simulator tier. None of them needs a backend. `npm run lint` and `npm run format:check` cover `fixture/` and `example/` alike. +`npm test` runs the type tests, the pure test tier and the `convex-test` tier. None of them needs a backend. `npm run lint` and `npm run format:check` cover `fixture/` and `example/` alike. -`npm run test:all` runs every project in one run, the native tier included, and its record holds every test of every project, each under the project that ran it. `npm run acceptance` then answers whether the first experiment passed: it derives the graph, takes the rows `acceptanceRows` of `spec:application.first-experiment` names, makes one required scenario per example of each row, and reads them against the newest record under `evidence/runs/`, or against the record `npm run acceptance -- ` names. It prints one line per required scenario with its status, `passed`, `failed`, `absent` or `missing`, and a last line `acceptance: passed` or `acceptance: not passed`. It exits 0 when every scenario passed in a passed run on a clean tree; 1 when a scenario failed or is absent, the run did not pass or the tree was not clean; 3 when nothing of that holds and a scenario has no example or no verifier; and 2 when it cannot answer. CI runs both and fails on 1 and 2. `spec:platform.acceptance-contract` holds the rules. +`npm run test:all` runs every project in one run, the native backend tier included, and its record holds every test of every project, each under the project that ran it. `npm run acceptance` then answers whether the first experiment passed: it derives the graph, takes the rows `acceptanceRows` of `spec:application.first-experiment` names, makes one required scenario per example of each row, and reads them against the newest record under `evidence/runs/`, or against the record `npm run acceptance -- ` names. It prints one line per required scenario with its status, `passed`, `failed`, `absent` or `missing`, and a last line `acceptance: passed` or `acceptance: not passed`. It exits 0 when every scenario passed in a passed run on a clean tree; 1 when a scenario failed or is absent, the run did not pass or the tree was not clean; 3 when nothing of that holds and a scenario has no example or no verifier; and 2 when it cannot answer. CI runs both and fails on 1 and 2. `spec:platform.acceptance-contract` holds the rules. -The native tier downloads the pinned backend release into `.cache/` on first use and checks the hash of the executable on every run. `harness/backend-release.json` pins the release for macOS on Apple silicon and for Linux on x86-64. On another platform, or to run another build, set `CONVEX_BACKEND_BINARY` to a `convex-local-backend` executable. The run's record then names the release only when the executable's hash is the pinned one. +The native backend tier downloads the pinned backend release into `.cache/` on first use and checks the hash of the executable on every run. `harness/backend-release.json` pins the release for macOS on Apple silicon and for Linux on x86-64. On another platform, or to run another build, set `CONVEX_BACKEND_BINARY` to a `convex-local-backend` executable. The run's record then names the release only when the executable's hash is the pinned one. The harness runs every Convex CLI call in the project directory of the composition it acts on, and gives the call the address and the admin key of the backend it started, and an environment of its own. A deploy key or a deployment named in your shell or in a `.env` file is never used. diff --git a/design/specs/advanced/coalesced-recomputation.source-changes-during-recompute.sdp.md b/design/specs/advanced/coalesced-recomputation.source-changes-during-recompute.sdp.md index 6c1e2b0..bd3e993 100644 --- a/design/specs/advanced/coalesced-recomputation.source-changes-during-recompute.sdp.md +++ b/design/specs/advanced/coalesced-recomputation.source-changes-during-recompute.sdp.md @@ -9,7 +9,7 @@ relations: --- # Source changes during a coalesced recompute -Sc L6-1 · native tier. +Sc L6-1 · native backend tier. ## Intent @@ -26,5 +26,5 @@ And the view reports its staleness ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts the recompute's clear conflicts under optimistic concurrency with the later mark or clears only the earlier one, and that a following recompute covers the change (D18, F1). diff --git a/design/specs/advanced/cross-stream-online-rebuild.backfill-races-live-writes.sdp.md b/design/specs/advanced/cross-stream-online-rebuild.backfill-races-live-writes.sdp.md index 79881ba..b7292b1 100644 --- a/design/specs/advanced/cross-stream-online-rebuild.backfill-races-live-writes.sdp.md +++ b/design/specs/advanced/cross-stream-online-rebuild.backfill-races-live-writes.sdp.md @@ -9,7 +9,7 @@ relations: --- # A cross-stream backfill races live writes and new subjects, then cuts over -Sc L6-4 · native tier. +Sc L6-4 · native backend tier. ## Intent @@ -25,5 +25,5 @@ And no command was run and no external effect was repeated by the rebuild ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts the switched generation equals a fold of every stream at the cut, that the old generation is kept for rollback, and that the obligation and provider stubs recorded nothing during the rebuild (D18, D9, Law 10). diff --git a/design/specs/advanced/ordered-consumer.duplicate-out-of-order-applied-once.sdp.md b/design/specs/advanced/ordered-consumer.duplicate-out-of-order-applied-once.sdp.md index c8d3c26..b604ffd 100644 --- a/design/specs/advanced/ordered-consumer.duplicate-out-of-order-applied-once.sdp.md +++ b/design/specs/advanced/ordered-consumer.duplicate-out-of-order-applied-once.sdp.md @@ -9,7 +9,7 @@ relations: --- # Duplicate and out-of-order events reach a noncommutative consumer -Sc L6-2 · native tier. +Sc L6-2 · native backend tier. ## Intent @@ -25,5 +25,5 @@ And the consumer position ends at sequence number {last: 5} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts the noncommutative view equals the result of applying the five events in order once, and that the worker waited for the missing head rather than applying a later event (D18). diff --git a/design/specs/advanced/ordered-consumer.partition-failure-isolated.sdp.md b/design/specs/advanced/ordered-consumer.partition-failure-isolated.sdp.md index 90f893d..aeb76b5 100644 --- a/design/specs/advanced/ordered-consumer.partition-failure-isolated.sdp.md +++ b/design/specs/advanced/ordered-consumer.partition-failure-isolated.sdp.md @@ -9,7 +9,7 @@ relations: --- # One ordered partition fails -Sc L6-3 · native tier. +Sc L6-3 · native backend tier. ## Intent @@ -25,5 +25,5 @@ And partition {failing: 2} stays blocked at its head with the failure inspectabl ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts the failed partition's position did not advance, that an unauthorized skip is refused, and that an authorized skip writes a recorded data-loss decision before the position advances (D18). diff --git a/design/specs/agents/agent-runs.budget-exceeded-blocks-execution.sdp.md b/design/specs/agents/agent-runs.budget-exceeded-blocks-execution.sdp.md index e503dc0..0ca3a33 100644 --- a/design/specs/agents/agent-runs.budget-exceeded-blocks-execution.sdp.md +++ b/design/specs/agents/agent-runs.budget-exceeded-blocks-execution.sdp.md @@ -9,7 +9,7 @@ relations: --- # The run exceeds its budget -Sc L5-2 · native tier · second of two cases: a run bound is exhausted. +Sc L5-2 · native backend tier · second of two cases: a run bound is exhausted. ## Intent @@ -25,5 +25,5 @@ And the run is marked exhausted with an operator exit ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test repeats the case for the attempts, elapsed time and call-count bounds and asserts each blocks execution and that no partial command commits (D17, Sc L5-2). diff --git a/design/specs/agents/agent-runs.changed-input-voids-approval.sdp.md b/design/specs/agents/agent-runs.changed-input-voids-approval.sdp.md index c265ced..a223923 100644 --- a/design/specs/agents/agent-runs.changed-input-voids-approval.sdp.md +++ b/design/specs/agents/agent-runs.changed-input-voids-approval.sdp.md @@ -9,7 +9,7 @@ relations: --- # Proposal input changes after approval -Sc L5-2 · native tier · first of two cases: the input changed after the approval was given. +Sc L5-2 · native backend tier · first of two cases: the input changed after the approval was given. ## Intent @@ -25,5 +25,5 @@ And the approval is void and the process record shows an explicit transition ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts the execution check compares the current input hash to the approval's hash before any context call, and that nothing commits (D17, D16, Law 5). diff --git a/design/specs/agents/agent-runs.unauthorized-proposal-rejected.sdp.md b/design/specs/agents/agent-runs.unauthorized-proposal-rejected.sdp.md index 6ca9d37..64e119b 100644 --- a/design/specs/agents/agent-runs.unauthorized-proposal-rejected.sdp.md +++ b/design/specs/agents/agent-runs.unauthorized-proposal-rejected.sdp.md @@ -9,7 +9,7 @@ relations: --- # An agent proposes a valid-looking unauthorized command -Sc L5-1 · native tier. +Sc L5-1 · native backend tier. ## Intent @@ -25,6 +25,6 @@ And no function exists that executes a proposal outside the command pipeline ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts the rejection is the same `ConvexError` code an unauthorized human would receive, and that no journal, state or receipt row was written (D17, Law 5, D7). - The test inspects the deployment's registered functions and asserts none takes a proposal and writes business state (D17, Law 1). diff --git a/design/specs/agents/agent-runs.untrusted-input-cannot-widen.sdp.md b/design/specs/agents/agent-runs.untrusted-input-cannot-widen.sdp.md index 67b1178..45fb352 100644 --- a/design/specs/agents/agent-runs.untrusted-input-cannot-widen.sdp.md +++ b/design/specs/agents/agent-runs.untrusted-input-cannot-widen.sdp.md @@ -9,7 +9,7 @@ relations: --- # A retrieved document or prompt tries to widen the agent's rights -Sc L5-4 · native tier. +Sc L5-4 · native backend tier. ## Intent @@ -25,5 +25,5 @@ And the run's grants after the run equal the grants before it ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts the grants table is unchanged, that no approval was created by the run, and that the caller namespace on every receipt is `agent` (D17, D11, Law 5). diff --git a/design/specs/agents/budget-accounting.timed-out-paid-call-settles-once.sdp.md b/design/specs/agents/budget-accounting.timed-out-paid-call-settles-once.sdp.md index 0bafcfd..c8a9541 100644 --- a/design/specs/agents/budget-accounting.timed-out-paid-call-settles-once.sdp.md +++ b/design/specs/agents/budget-accounting.timed-out-paid-call-settles-once.sdp.md @@ -9,7 +9,7 @@ relations: --- # A paid call times out and usage arrives later -Sc L5-3 · native tier. +Sc L5-3 · native backend tier. ## Intent @@ -26,6 +26,6 @@ And the slot is released by reconciliation or by the operator exit, never lost f ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts that between the timeout and the report the reservation shows unknown cost, not zero, and that a second copy of the same usage report is refused (D17, Law 7). - The test asserts the slot is held until reconciliation and that the operator exit releases it with a repair record when no report ever arrives (D17, D13). diff --git a/design/specs/application/first-experiment.native-acceptance-production-configuration.sdp.md b/design/specs/application/first-experiment.native-acceptance-production-configuration.sdp.md index b5bd699..3a2f4f3 100644 --- a/design/specs/application/first-experiment.native-acceptance-production-configuration.sdp.md +++ b/design/specs/application/first-experiment.native-acceptance-production-configuration.sdp.md @@ -10,7 +10,7 @@ relations: --- # Run native acceptance with production configuration -Sc L2-9 · end to end tier · production composition. +Sc L2-9 · native backend tier · production composition. ## Intent @@ -25,7 +25,7 @@ Then authority, schemas and code path {parity: "match a release"} ## Verification — executable -- Runs in the end-to-end tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - The row is read as the local observation, the production composition on the pinned local backend with the same authority, schemas, concurrency and code path a release has, and this example claims no parity with a hosted deployment; it binds no concurrency claim, because the row's concurrency half is the contention run of `spec:application.first-experiment` on the production composition. - This example is the end-to-end path alone, and its test runs no other scenario: the harness deploys `example/` once, with its own `auth.config.ts`, grants, namespaces, entry points and schema, and the test sends, through ordinary clients and one call after another, `ReceiveStock` and `PlaceOrder` applied, a `PlaceOrder` rejected `insufficientStock`, a `PlaceOrder` and a `getOrder` refused `forbidden` to a caller with no grant, a `getOrder` refused `unauthenticated` to a caller with no identity, `getOrder`, `listOrders` and `listOrderSummaries` answered to a granted caller, and each internal function of the composition, named directly, refused as not public. No two calls of the path run at once. Whether every scenario routed to the production composition passed there is the acceptance check's to answer, under `spec:platform.acceptance-contract`. - The test asserts that the function log, from a mark before the path to its last call, holds one top-level query or mutation record for each call of the path and no record of any other function, that each refused internal function read no document, and that the applied order's receipt, its order and stock item stream rows and its order summary row are stored and the refused calls stored nothing. diff --git a/design/specs/application/first-experiment.native-contention.sdp.md b/design/specs/application/first-experiment.native-contention.sdp.md index cb9d5c2..4a9dafb 100644 --- a/design/specs/application/first-experiment.native-contention.sdp.md +++ b/design/specs/application/first-experiment.native-contention.sdp.md @@ -9,7 +9,7 @@ relations: --- # Concurrent commands on the production composition -Sc L2-9 · native tier · production composition · Traces: First experiment, Sc L1-11, Sc L2-9, E-47. +Sc L2-9 · native backend tier · production composition · Traces: First experiment, Sc L1-11, Sc L2-9, E-47. The concurrency half of native acceptance observes the production composition on the pinned local backend. The separate production-configuration example carries the code-path half. Neither claims parity with a hosted deployment. diff --git a/design/specs/application/first-experiment.order-at-stock-item-id-bound.sdp.md b/design/specs/application/first-experiment.order-at-stock-item-id-bound.sdp.md index 4a9ef52..adca455 100644 --- a/design/specs/application/first-experiment.order-at-stock-item-id-bound.sdp.md +++ b/design/specs/application/first-experiment.order-at-stock-item-id-bound.sdp.md @@ -9,7 +9,7 @@ relations: --- # An order of the maximum number of lines at the stock item ID bound -E-46 · native tier · production composition · no acceptance row; the example verifies that the maximum order fits the event payload bound when every stock item ID is at the bound `PlaceOrder`'s declaration carries. +E-46 · native backend tier · production composition · no acceptance row; the example verifies that the maximum order fits the event payload bound when every stock item ID is at the bound `PlaceOrder`'s declaration carries. ## Intent @@ -28,6 +28,6 @@ And the order's OrderPlaced payload measures {payloadBytes: 11725} bytes ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - After grants and the order summary's first rebuild through to its switch as setup, the test receives stock for the order's 100 stock items through `ReceiveStock`, each ID distinct and exactly 64 bytes of UTF-8, and sends one `PlaceOrder` with one line on each from an ordinary client under a request key. - It asserts the applied response, reads the documents read and written from the command's own top-level completion record's `usageStats`, and measures with `getConvexSize` the payload of the one `OrderPlaced` event the Orders context's journal holds for the order, read through admin access. diff --git a/design/specs/application/first-experiment.order-of-1-line.sdp.md b/design/specs/application/first-experiment.order-of-1-line.sdp.md index 1fa4d0f..c79ab51 100644 --- a/design/specs/application/first-experiment.order-of-1-line.sdp.md +++ b/design/specs/application/first-experiment.order-of-1-line.sdp.md @@ -9,7 +9,7 @@ relations: --- # An order of 1 line -Sc L2-3 · native tier · production composition · the first of the three enumerated sizes. +Sc L2-3 · native backend tier · production composition · the first of the three enumerated sizes. ## Intent @@ -28,7 +28,7 @@ And the budgets {budgets: "hold"} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends `PlaceOrder` with one line, reads the backend's function log from a mark before the command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for the command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation for this size. - The test reads the documents and bytes read and written from the completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts six documents written: the receipt, the order's state and event, the stock item's state and event, and the summary row; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-10-lines.sdp.md b/design/specs/application/first-experiment.order-of-10-lines.sdp.md index bcf2c23..0694cd7 100644 --- a/design/specs/application/first-experiment.order-of-10-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-10-lines.sdp.md @@ -9,7 +9,7 @@ relations: --- # An order of 10 lines -Sc L2-3 · native tier · production composition · the second of the three enumerated sizes. +Sc L2-3 · native backend tier · production composition · the second of the three enumerated sizes. ## Intent @@ -28,7 +28,7 @@ And the budgets {budgets: "hold"} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` with ten lines over ten stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation carrying the order and then one on the Inventory operation carrying all ten lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts that the ten-line order wrote two documents more per added line than the one-line reference, the added stock item's state and its event, so documents written grew linearly with the lines; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-max-lines.sdp.md b/design/specs/application/first-experiment.order-of-max-lines.sdp.md index d7b6c90..bf2e288 100644 --- a/design/specs/application/first-experiment.order-of-max-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-max-lines.sdp.md @@ -9,7 +9,7 @@ relations: --- # An order of the maximum number of lines -Sc L2-3 · native tier · production composition · the third of the three enumerated sizes; the maximum is 100 lines, the bound `PlaceOrder`'s declaration carries. +Sc L2-3 · native backend tier · production composition · the third of the three enumerated sizes; the maximum is 100 lines, the bound `PlaceOrder`'s declaration carries. ## Intent @@ -28,7 +28,7 @@ And the budgets {budgets: "hold"} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row; and it asserts that the execution time the completion record reports is under 1 second, a bound this example sets on that reported time and not the engine's cap on completion time, because the engine's limit is on the function's own computation and nested calls draw on a separate budget. - The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation carrying all 100 lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, and asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.receive-stock-item-id-past-bound-refused.sdp.md b/design/specs/application/first-experiment.receive-stock-item-id-past-bound-refused.sdp.md index 6f86294..9fe2038 100644 --- a/design/specs/application/first-experiment.receive-stock-item-id-past-bound-refused.sdp.md +++ b/design/specs/application/first-experiment.receive-stock-item-id-past-bound-refused.sdp.md @@ -9,7 +9,7 @@ relations: --- # ReceiveStock refuses a stock item ID one byte past its bound -E-46 · native tier · production composition · no acceptance row; the example verifies that `ReceiveStock` refuses a stock item ID longer than the bound `PlaceOrder` also carries at step 1, before any read and before the Inventory context is called. +E-46 · native backend tier · production composition · no acceptance row; the example verifies that `ReceiveStock` refuses a stock item ID longer than the bound `PlaceOrder` also carries at step 1, before any read and before the Inventory context is called. ## Intent @@ -27,6 +27,6 @@ And the command read {readDocuments: 0} documents and wrote {writtenDocuments: 0 ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - After grants and the order summary's first rebuild through to its switch as setup, the test sends one `ReceiveStock` of 100 items with distinct stock item IDs from an ordinary client under a request key, the last ID built from multi-byte UTF-8 characters so that its character count is below its byte count. -- It asserts the `ConvexError` data exactly, `{ kind: "rejection", code: "invalidInput", commandType: "ReceiveStock", message: "Line 99 needs a stock item ID of at most 64 bytes of UTF-8, not 65", details: { line: 99, length: 65, limit: 64 } }`, reads zero documents read and written from the command's own top-level completion record's `usageStats`, and asserts that the receipts and the Inventory context's stream rows and events are unchanged. +- It asserts the `ConvexError` data exactly, `{ kind: "rejection", code: "invalidInput", entry: "ReceiveStock", message: "Line 99 needs a stock item ID of at most 64 bytes of UTF-8, not 65", details: { line: 99, length: 65, limit: 64 } }`, reads zero documents read and written from the command's own top-level completion record's `usageStats`, and asserts that the receipts and the Inventory context's stream rows and events are unchanged. diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 8a88c4a..5923bf2 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -76,7 +76,7 @@ The six cost targets are the constraint Specs this workflow is constrained by. T - rule: The results feed the open question on component cost and the Layer 3 decisions (First experiment, OQ1, Decision method rule 4) - rule: Native acceptance runs with production configuration, read as the local observation: the production composition on the pinned local backend, with the same authority, schemas, concurrency and code path a release has; no pass of the experiment claims parity with a hosted deployment (Sc L2-9) - rule: The concurrency half of Sc L2-9 is the contention run on the production composition, and a run on a hosted deployment is deferred together with the function-call quota half of Probe 3 (Sc L2-9, Probe 3) -- rule: [extension] The native tier obtains its identities from tokens the harness signs for the fixture issuer that the deployment's environment variables name, carried by ordinary clients, so the actor mapping, the grants, the namespaces, the entry points and Convex's visibility check are the production ones; the issuer is the identity source, the one named difference from production, and every run's evidence record states it (E-13, S13, Sc L2-9) +- rule: [extension] The native backend tier obtains its identities from tokens the harness signs for the fixture issuer that the deployment's environment variables name, carried by ordinary clients, so the actor mapping, the grants, the namespaces, the entry points and Convex's visibility check are the production ones; the issuer is the identity source, the one named difference from production, and every run's evidence record states it (E-13, S13, Sc L2-9) - rule: Every run is recorded as acceptance evidence: commit, installed layers, backend and dependency versions, configuration, dataset, command and result, and a run with adjusted configuration states how it differs from production (Acceptance scenarios) - rule: The kernel's fixture app is separate from the example app, and test-only functions never ship (Acceptance scenarios) - rule: [extension] The second lifecycle command is `CancelOrder` and the essential summary is the order summary read model (E-46, First experiment) diff --git a/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md b/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md index 78c30f2..8d6cb41 100644 --- a/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md +++ b/design/specs/application/first-experiment.stock-item-id-past-bound-refused.sdp.md @@ -9,7 +9,7 @@ relations: --- # A stock item ID one byte past its bound is refused -E-46 · native tier · production composition · no acceptance row; the example verifies that `PlaceOrder` refuses a stock item ID longer than its bound at step 1, before any read and before either context is called. +E-46 · native backend tier · production composition · no acceptance row; the example verifies that `PlaceOrder` refuses a stock item ID longer than its bound at step 1, before any read and before either context is called. ## Intent @@ -28,6 +28,6 @@ And the command read {readDocuments: 0} documents and wrote {writtenDocuments: 0 ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - After grants and the order summary's first rebuild through to its switch as setup, the test receives stock through `ReceiveStock` for the 99 stock items of the order within the bound, the last one being an ID `ReceiveStock` refuses too, so a refusal later than step 1 would be short stock and not `invalidInput`, and sends one `PlaceOrder` with one line on each from an ordinary client under a request key. -- It asserts the `ConvexError` data exactly, `{ kind: "rejection", code: "invalidInput", commandType: "PlaceOrder", message: "Line 99 needs a stock item ID of at most 64 bytes of UTF-8, not 65", details: { line: 99, length: 65, limit: 64 } }`, reads zero documents read and written from the command's own top-level completion record's `usageStats`, which places the refusal before the grant read and before either context's sub-transaction, and asserts that the receipts, both contexts' stream rows and events and the order summaries are unchanged. +- It asserts the `ConvexError` data exactly, `{ kind: "rejection", code: "invalidInput", entry: "PlaceOrder", message: "Line 99 needs a stock item ID of at most 64 bytes of UTF-8, not 65", details: { line: 99, length: 65, limit: 64 } }`, reads zero documents read and written from the command's own top-level completion record's `usageStats`, which places the refusal before the grant read and before either context's sub-transaction, and asserts that the receipts, both contexts' stream rows and events and the order summaries are unchanged. diff --git a/design/specs/application/generation-registry.command-without-generation-fails.sdp.md b/design/specs/application/generation-registry.command-without-generation-fails.sdp.md index 37f649e..c7e8338 100644 --- a/design/specs/application/generation-registry.command-without-generation-fails.sdp.md +++ b/design/specs/application/generation-registry.command-without-generation-fails.sdp.md @@ -9,7 +9,7 @@ relations: --- # A command whose read model has no generation fails -E-8 · native tier · production composition · no acceptance row; the example verifies that a command never commits without its read model, an extension under E-8. +E-8 · native backend tier · production composition · no acceptance row; the example verifies that a command never commits without its read model, an extension under E-8. ## Intent @@ -24,7 +24,7 @@ And the number of stored receipts, events and state changes is {stored: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - The test creates stock through `ReceiveStock`, which declares no read model, and sends `PlaceOrder` before any generation of the order summary is started. - The test asserts that the caller's error is not a `ConvexError` of the outcome boundary, that the function log's record of the failure names the order summary, and that no receipt, no order stream, no order event and no change to the stock exists for the operation. - The test then starts generation 1 of the order summary, re-sends the same command with the same request key while the generation is still being built, and asserts that it is applied and that its row is in generation 1. diff --git a/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md b/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md index 83ac714..c748b32 100644 --- a/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md +++ b/design/specs/application/generation-registry.first-rebuild-installs-read-model.sdp.md @@ -9,7 +9,7 @@ relations: --- # A read model is installed through its first rebuild -E-8 · native tier · fixture composition · no acceptance row; the example verifies that a read model's first generation is built like any other and serves the subjects that already exist, an extension under E-8. +E-8 · native backend tier · fixture composition · no acceptance row; the example verifies that a read model's first generation is built like any other and serves the subjects that already exist, an extension under E-8. ## Intent @@ -25,7 +25,7 @@ And a second start while generation 1 is being built is {second: "refused"} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test creates three documents through `CreateDocument`, which declares no read model, then runs `internal.rebuild.startGeneration` for `documentSummary` with admin access and a stated operator, and asserts one row in `generations`, generation 1, `building`, with `documentSummary`'s latest projection version and the operator as `startedBy`. - In this order the test closes the tenant's gate, runs the first start, reads the generation row `building`, runs `startGeneration` again and asserts that its error names the read model, generation 1 and its state and that the table still holds one row, waits until the first batch has parked under the gate, reopens the gate and runs `resumeGeneration`. - The test waits until the row is `verified`, asserts that `listDocumentSummaries` still fails for want of an active generation, runs `switchGeneration`, and asserts that the row is `active`, that no row is `retired`, and that `listDocumentSummaries` returns the three documents under their status. diff --git a/design/specs/application/orders-inventory-example.cancel-duplicate-answered-from-receipt.sdp.md b/design/specs/application/orders-inventory-example.cancel-duplicate-answered-from-receipt.sdp.md index 1e789ec..7d24cbc 100644 --- a/design/specs/application/orders-inventory-example.cancel-duplicate-answered-from-receipt.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-duplicate-answered-from-receipt.sdp.md @@ -9,7 +9,7 @@ relations: --- # A repeated cancel request is answered from its receipt -E-46 · native tier · production composition · no acceptance row; the example verifies that `CancelOrder` is answered from its receipt when its request is repeated. +E-46 · native backend tier · production composition · no acceptance row; the example verifies that `CancelOrder` is answered from its receipt when its request is repeated. ## Intent @@ -28,5 +28,5 @@ And the number of documents the command wrote is {written: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - The test asserts that the response is `{ kind: "applied", replayed: true, result: null }` with the first cancel's `operationId` and versions, and that the receipts table holds one receipt for `k-1`. diff --git a/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md b/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md index 81e4d8a..003c98a 100644 --- a/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md @@ -9,7 +9,7 @@ relations: --- # A placed order is cancelled -E-46 · native tier · production composition · no acceptance row; the example verifies `CancelOrder`, the example domain's second lifecycle command under E-46. +E-46 · native backend tier · production composition · no acceptance row; the example verifies `CancelOrder`, the example domain's second lifecycle command under E-46. ## Intent @@ -27,7 +27,7 @@ And the number of documents the command wrote is {written: 6} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - The order is `order-1` with two lines on the stock item `sku-1`, of 2 and 1 units; the test asserts that the stock item's state after the cancel equals its state before `PlaceOrder`, `{ onHand: 5, allocated: 0 }`. - The test asserts the response `{ kind: "applied", replayed: false, result: { orderId: "order-1", released: [{ stockItemId: "sku-1", quantity: 3 }] } }` with versions naming the order at version 2 and the stock item at version 3, Orders first; the order of the two calls, Orders and then Inventory, is shown at the pure tier by the same executor run against a ctx that records each `runMutation` by its function reference, which this native test cannot tell apart. - The six documents are the receipt, the order's state and its `OrderCancelled` event, the stock item's state and its one `AllocationReleased` event of quantity 3, and the summary row, whose `sourceVersions` name the order at version 2; the parent's `getOrder` answers the order with status `cancelled` and the lines, total and `placedAt` it answered before the cancel, and the summary row keeps that `placedAt`. diff --git a/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md b/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md index b929917..63a554c 100644 --- a/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md @@ -9,7 +9,7 @@ relations: --- # Orders cancels, then Inventory refuses the release -E-46 · native tier · production composition · no acceptance row; the example verifies that a release Inventory refuses rolls back the cancel Orders recorded, which is Sc L2-1's rule on the second lifecycle command. +E-46 · native backend tier · production composition · no acceptance row; the example verifies that a release Inventory refuses rolls back the cancel Orders recorded, which is Sc L2-1's rule on the second lifecycle command. ## Intent @@ -29,7 +29,7 @@ And the number of documents the command wrote is {written: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - No public command releases stock without cancelling its order, so the setup calls the Inventory context's release operation for the order's 3 units directly, with admin access, as a sanctioned operation of that context; the stock item's state stays the fold of its events, and the two contexts now disagree about the order. -- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "insufficientAllocation", commandType: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 } }`; that the release's quantities come from the lines the Orders call returned, and that Orders is called first, is shown at the pure tier by the same executor run against a ctx that records each `runMutation` by its function reference. +- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "insufficientAllocation", entry: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 } }`; that the release's quantities come from the lines the Orders call returned, and that Orders is called first, is shown at the pure tier by the same executor run against a ctx that records each `runMutation` by its function reference. - The test reads the function log and asserts that the request ran one top-level mutation, recorded as failed with no document written, and that the order's stream stays at version 1 with status `placed` and one event. diff --git a/design/specs/application/orders-inventory-example.cancel-twice-rejected.sdp.md b/design/specs/application/orders-inventory-example.cancel-twice-rejected.sdp.md index f6ea881..9928e93 100644 --- a/design/specs/application/orders-inventory-example.cancel-twice-rejected.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-twice-rejected.sdp.md @@ -9,7 +9,7 @@ relations: --- # A second cancel of one order is rejected -E-46 · native tier · production composition · no acceptance row; the example verifies that the order's own state refuses a second cancel. +E-46 · native backend tier · production composition · no acceptance row; the example verifies that the order's own state refuses a second cancel. ## Intent @@ -29,6 +29,6 @@ And the number of documents the command wrote is {written: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. -- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "orderAlreadyCancelled", commandType: "CancelOrder", message: "The order is already cancelled" }`, and that no receipt exists for `k-2`. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. +- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "orderAlreadyCancelled", entry: "CancelOrder", message: "The order is already cancelled" }`, and that no receipt exists for `k-2`. - The stock item's state stays `{ onHand: 5, allocated: 0 }`; had the second cancel reached Inventory, its release of 3 would have been refused `insufficientAllocation`, so the code `orderAlreadyCancelled` shows that Orders refused it first. diff --git a/design/specs/application/orders-inventory-example.cancel-unknown-order-rejected.sdp.md b/design/specs/application/orders-inventory-example.cancel-unknown-order-rejected.sdp.md index 6d6fea1..da0fbde 100644 --- a/design/specs/application/orders-inventory-example.cancel-unknown-order-rejected.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-unknown-order-rejected.sdp.md @@ -9,7 +9,7 @@ relations: --- # A cancel of an order never placed is rejected -E-46 · native tier · production composition · no acceptance row; the example verifies that the order decider refuses a cancel of an order with no event. +E-46 · native backend tier · production composition · no acceptance row; the example verifies that the order decider refuses a cancel of an order with no event. ## Intent @@ -28,6 +28,6 @@ And the number of documents the command wrote is {written: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. -- The order never placed is `order-2`; the test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "orderNotFound", commandType: "CancelOrder", message: "The order does not exist" }`, that the Orders context holds no stream row and no event for `order-2`, and that the parent's `getOrder` answers null for it. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. +- The order never placed is `order-2`; the test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "orderNotFound", entry: "CancelOrder", message: "The order does not exist" }`, that the Orders context holds no stream row and no event for `order-2`, and that the parent's `getOrder` answers null for it. - The order summary's status is that of the placed order `order-1`, the one row the table holds. diff --git a/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md b/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md index 5476c1c..59cf92b 100644 --- a/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-without-grant-refused.sdp.md @@ -9,7 +9,7 @@ relations: --- # A caller without the cancel grant is refused -E-46 · native tier · production composition · no acceptance row; the example verifies that `CancelOrder` requires its own permission, under Law 5 and D11. +E-46 · native backend tier · production composition · no acceptance row; the example verifies that `CancelOrder` requires its own permission, under Law 5 and D11. ## Intent @@ -28,6 +28,6 @@ And the number of documents the command wrote is {written: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. -- The refused caller is a second user of the fixture issuer holding `orders.place`, `inventory.receive`, `orders.read` and `inventory.read` in the tenant; the test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "forbidden", commandType: "CancelOrder", message: "The caller may not run CancelOrder in this tenant", details: { reason: "no_grant" } }`. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. +- The refused caller is a second user of the fixture issuer holding `orders.place`, `inventory.receive`, `orders.read` and `inventory.read` in the tenant; the test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "forbidden", entry: "CancelOrder", message: "The caller may not run CancelOrder in this tenant", details: { reason: "no_grant" } }`. - The refused caller then sends `CancelOrder` for `order-2`, an order never placed, and the test asserts the same `forbidden` data, which shows the refusal comes before Orders decides: an execution before the check would answer `orderNotFound`. diff --git a/design/specs/application/parent-use-cases.sdp.md b/design/specs/application/parent-use-cases.sdp.md index 2532bbd..c9b3d5a 100644 --- a/design/specs/application/parent-use-cases.sdp.md +++ b/design/specs/application/parent-use-cases.sdp.md @@ -105,7 +105,7 @@ A use case is declared with the command declaration of `spec:command.command-dec - createdDetection: a stream created by the call is one the adapter loaded without a row, which its `StreamResult` records as `created` and the operation outcome carries on that stream's `streams` entry; the pipeline's step 9 selects the projection's live-created mode from that flag and the live-updated mode otherwise, so no use case derives the flag from a version count (D8, D9, E-9, E-23) - typeContextCall: `ctx.runMutation(components.inventory.operations.allocate, { tenantId, actor, operation, input: { orderId, lines }, facts })` returns the `OperationOutcome` of `spec:context.persistence-adapter`, the kernel's `CommittedOutcome` with the DTO and `versions: StreamVersion[]` plus `streams`; `operation` is the `OperationRef` the pipeline minted, and the `args` shape is the one `spec:context.context-component` fixes (D10, F2, F11, E-22, E-23) - typeBusinessFailurePolicy: `[extension] type BusinessFailurePolicy = { kind: "carry" } | { kind: "reject"; code: string; message: string }` declared per use case and per context call as a constant beside the executor; `carry` returns the context's business failure as the use case's, `reject` throws the named rejection code with the stated message, and the declaration's `rejections` list must contain the code (E-44, D4) -- fnCarryOrReject: `[extension] carryOrReject(outcome: OperationOutcome, policy: BusinessFailurePolicy): OperationOutcome` the one place the policy is applied: it returns `outcome` unchanged when its `kind` is `applied` or the policy is `carry`, and otherwise throws the kernel's bare `Rejection`, `new ConvexError({ code: policy.code, message: policy.message })`, the shape a context throws, so the pipeline's `normalizeThrown` adds `kind` and the command type and turns a code the declaration does not list into a technical failure; it never calls the outcome boundary's `reject`, which needs a command type the helper does not have (E-44, D4, D7) +- fnCarryOrReject: `[extension] carryOrReject(outcome: OperationOutcome, policy: BusinessFailurePolicy): OperationOutcome` the one place the policy is applied: it returns `outcome` unchanged when its `kind` is `applied` or the policy is `carry`, and otherwise throws the kernel's bare `Rejection`, `new ConvexError({ code: policy.code, message: policy.message })`, the shape a context throws, so the pipeline's `normalizeThrown` adds `kind` and the entry and turns a code the declaration does not list into a technical failure; it never calls the outcome boundary's `reject`, which needs an `entry` the helper does not have (E-44, D4, D7) - useCaseRejection: [extension] a rejection the executor's own code raises, from a rule that links two contexts, is thrown the same way, `throw new ConvexError({ code, message, details })`, never through `reject`, so every rejection of a use case, a context's or its own, reaches the caller through the one rethrow and the one check of the code list (D7, D10, E-5) - errorCodeOperationTooLarge: `operationTooLarge`, the kernel's reserved code the outcome boundary lists, thrown as a rejection by the pipeline's step 1 when the input exceeds the declaration's `bounds`, and by a context's adapter when a list exceeds its own `maxStreams` or byte bound; it carries the bound and the offered size (D10, F13, E-38) - tableImports: `[extension] imports: defineTable({ tenantId: v.string(), importId: v.string(), commandType: v.string(), operationId: v.string(), expected: v.number(), done: v.number(), failed: v.number(), state: v.union(v.literal("running"), v.literal("completed"), v.literal("failed"), v.literal("cancelled")), lastBatch: v.optional(v.string()), startedAt: v.number(), updatedAt: v.number(), startedBy: actorValidator })` (E-41, D10, Law 11) diff --git a/design/specs/application/parent-use-cases.second-context-rejects.sdp.md b/design/specs/application/parent-use-cases.second-context-rejects.sdp.md index 552a82b..2295770 100644 --- a/design/specs/application/parent-use-cases.second-context-rejects.sdp.md +++ b/design/specs/application/parent-use-cases.second-context-rejects.sdp.md @@ -9,7 +9,7 @@ relations: --- # The first context writes, then the second rejects -Sc L2-1 · native tier · production composition · the rejection case. +Sc L2-1 · native backend tier · production composition · the rejection case. ## Intent @@ -26,8 +26,8 @@ And the number of stored receipts, events and state changes is {stored: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - The test creates stock through `ReceiveStock`, then sends `PlaceOrder` with a line that asks for more than is available, so the Orders context's call returns before the Inventory context throws `insufficientStock`. -- The test asserts that the caller's error is a `ConvexError` whose data has `kind` `rejection`, `code` `insufficientStock` and `commandType` `PlaceOrder`, that no receipt row exists for the request key, that the Orders context has no stream row and no event for the order, that no order summary row exists, and that no document of the parent or of either context changed. +- The test asserts that the caller's error is a `ConvexError` whose data has `kind` `rejection`, `code` `insufficientStock` and `entry` `PlaceOrder`, that no receipt row exists for the request key, that the Orders context has no stream row and no event for the order, that no order summary row exists, and that no document of the parent or of either context changed. - The test reads the function log and asserts that the request ran one top-level mutation, recorded as failed with no document written. - The test then receives the missing quantity through `ReceiveStock`, re-sends the same command and asserts that it is applied, as one top-level mutation whose returned versions name both contexts, which shows the first attempt left nothing behind. diff --git a/design/specs/application/parent-use-cases.second-context-throws.sdp.md b/design/specs/application/parent-use-cases.second-context-throws.sdp.md index caa477f..a5570b6 100644 --- a/design/specs/application/parent-use-cases.second-context-throws.sdp.md +++ b/design/specs/application/parent-use-cases.second-context-throws.sdp.md @@ -9,7 +9,7 @@ relations: --- # The first context writes, then the second throws -Sc L2-1 · native tier · fixture composition · the technical-failure case, the doc's second enumerated case of the row. +Sc L2-1 · native backend tier · fixture composition · the technical-failure case, the doc's second enumerated case of the row. The fault needs a part the fixture composition supplies, so the scenario runs there: the fixture command `FileDocument` calls the `depot` context, which creates a document, and then the second fixture context, `yard`, to file a copy that names the document version the `depot` call returned; the `yard` operation throws a plain Error after its own state write, through the fault its stream registration carries, the copy stream's `toDto` refusing the fault title. The libraries carry no hook. @@ -28,7 +28,7 @@ And the number of stored receipts, events and state changes is {stored: 0} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - An ordinary client with a grant sends `FileDocument` with the title that makes the `yard` context's save fail, after the `depot` context's call has returned its versions; the test asserts that the thrown error names version 1 of the document, which only the completed `depot` call returned. - The test asserts that the caller sees a technical failure and no rejection data, that the function log holds one completion record for the request, the parent mutation's own, carrying the fault's error, and that no receipt, no event and no stream row of either context exists for the operation. - The test then sends the same command without the fault, under the same request key, and asserts that it is applied with one version in each of the two contexts, because nothing of the first attempt survived. diff --git a/design/specs/application/read-models.committed-state-read-through-query.sdp.md b/design/specs/application/read-models.committed-state-read-through-query.sdp.md index 643aabf..2b4ade7 100644 --- a/design/specs/application/read-models.committed-state-read-through-query.sdp.md +++ b/design/specs/application/read-models.committed-state-read-through-query.sdp.md @@ -9,7 +9,7 @@ relations: --- # A successful command, then a query -Sc L2-2 · native tier · production composition · the query case, the doc's first enumerated case of the row. +Sc L2-2 · native backend tier · production composition · the query case, the doc's first enumerated case of the row. ## Intent @@ -24,6 +24,6 @@ And the number of workers, jobs or queues that ran is {workers: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - After grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test runs `PlaceOrder` and then, from the same client, calls the parent query `getOrder`, which relays the Orders context's `get`, and asserts that the DTO it returns holds the order's committed state and that its `version` equals the command's returned `versions` entry for the order's stream. - The test asserts that no scheduled function, cron or action ran for the command by reading the disposable backend's function log from a mark before the command to the `getOrder` read that the same client sends after the command returned, a window an ordinary read closes, and finding no record that a client did not cause; and it asserts that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either. diff --git a/design/specs/application/read-models.committed-state-visible.sdp.md b/design/specs/application/read-models.committed-state-visible.sdp.md index 1211433..3b8beda 100644 --- a/design/specs/application/read-models.committed-state-visible.sdp.md +++ b/design/specs/application/read-models.committed-state-visible.sdp.md @@ -9,7 +9,7 @@ relations: --- # A successful command, then a query or subscription -Sc L2-2 · native tier · production composition. +Sc L2-2 · native backend tier · production composition. ## Intent @@ -25,6 +25,6 @@ And the number of workers, jobs or queues that ran is {workers: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - After grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test subscribes to `listOrderSummaries` with the tenant, the status a placed order has and `{ cursor: null, numItems: 10 }`, runs `PlaceOrder`, and asserts that the next subscription value's page carries the order's summary whose `sourceVersions` equal the command's returned `versions` entry for the order's stream, the one stream the summary is projected from. - The test asserts that the backend ran no scheduled function, cron or action between the command and the subscription update, by reading the disposable backend's function log from before the command until it holds the command's mutation and the subscription's query, a window an ordinary read after the command closes, and finding no record that a client did not cause; and it asserts that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either. diff --git a/design/specs/application/read-models.list-pages-by-cursor.sdp.md b/design/specs/application/read-models.list-pages-by-cursor.sdp.md index 9c206fc..1169c5f 100644 --- a/design/specs/application/read-models.list-pages-by-cursor.sdp.md +++ b/design/specs/application/read-models.list-pages-by-cursor.sdp.md @@ -9,7 +9,7 @@ relations: --- # A client reads a tenant's order list in pages by cursor -E-24 · native tier · production composition · no acceptance row; the example verifies paging by an explicit cursor pair and the row cap of twice the item cap, an extension under E-24. +E-24 · native backend tier · production composition · no acceptance row; the example verifies paging by an explicit cursor pair and the row cap of twice the item cap, an extension under E-24. ## Intent @@ -25,6 +25,6 @@ And the number of pages that carry pageStatus SplitRequired is {splitPages: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - After grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test places the orders through `PlaceOrder`, then reads `listOrders` for the first tenant as a caller granted `orders.read`, each page with `cursor` and `numItems` and then subscribed with `endCursor` set to the `continueCursor` that read returned, the next page starting at the previous page's `endCursor`, until a read answers `isDone`; with every page subscribed, it places two orders whose order IDs sort inside the first page, `order-00a` and `order-05a`. - The test asserts that the first page, pinned by its end cursor, still starts and ends at the orders it held and grows from 10 to 12 orders, that the second page starts at the order after it, that the pages then hold 12, 10 and 5 orders, that no page carries `pageStatus` `SplitRequired`, and that every order ID of the first tenant appears once, none missing or repeated. diff --git a/design/specs/application/read-models.query-refused-before-disclosure.sdp.md b/design/specs/application/read-models.query-refused-before-disclosure.sdp.md index ae5944e..7786ca7 100644 --- a/design/specs/application/read-models.query-refused-before-disclosure.sdp.md +++ b/design/specs/application/read-models.query-refused-before-disclosure.sdp.md @@ -9,7 +9,7 @@ relations: --- # A caller with no identity reads an order through the parent -Law 5 · native tier · production composition · no acceptance row; the first of two refusals of a parent query, the caller with no identity. +Law 5 · native backend tier · production composition · no acceptance row; the first of two refusals of a parent query, the caller with no identity. ## Intent @@ -25,7 +25,7 @@ And the number of receipts the read wrote is {receipts: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. -- The test calls `getOrder` from an HTTP client with no token and subscribes to it from a client with no token, reads `error.data` from the call and from the subscription's error callback, and asserts for both that `kind` is `rejection`, `code` is `unauthenticated` and `commandType` is `getOrder`, and that `classifyThrown` answers `rejection`. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. +- The test calls `getOrder` from an HTTP client with no token and subscribes to it from a client with no token, reads `error.data` from the call and from the subscription's error callback, and asserts for both that `kind` is `rejection`, `code` is `unauthenticated` and `entry` is `getOrder`, and that `classifyThrown` answers `rejection`. - The test asserts that the receipts table holds no row written by the read. - The test then calls `getOrder` as a caller granted `orders.read` in the tenant and asserts that it returns the order's DTO. diff --git a/design/specs/application/read-models.query-refused-without-grant.sdp.md b/design/specs/application/read-models.query-refused-without-grant.sdp.md index 63ee8e6..c99784a 100644 --- a/design/specs/application/read-models.query-refused-without-grant.sdp.md +++ b/design/specs/application/read-models.query-refused-without-grant.sdp.md @@ -9,7 +9,7 @@ relations: --- # A caller with no grant in the tenant reads an order through the parent -Law 5 · native tier · production composition · no acceptance row; the second of two refusals of a parent query, the caller with an identity and no grant. +Law 5 · native backend tier · production composition · no acceptance row; the second of two refusals of a parent query, the caller with an identity and no grant. ## Intent @@ -25,7 +25,7 @@ And the number of receipts the read wrote is {receipts: 0} ## Verification — executable -- Runs in the native tier on the production composition; every test owns its disposable backend. -- The test calls `getOrder` as a caller whose token is valid and who holds no grant in the order's tenant, and asserts that `error.data` has `kind` `rejection`, `code` `forbidden`, `commandType` `getOrder` and `details.reason` `no_grant`, and that `classifyThrown` answers `rejection`. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. +- The test calls `getOrder` as a caller whose token is valid and who holds no grant in the order's tenant, and asserts that `error.data` has `kind` `rejection`, `code` `forbidden`, `entry` `getOrder` and `details.reason` `no_grant`, and that `classifyThrown` answers `rejection`. - The test asserts that the receipts table holds no row written by the read. - The test then grants the same caller `orders.read` in the tenant with admin access, calls `getOrder` again and asserts that it returns the order's DTO. diff --git a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md index 27b01a3..c72d1e8 100644 --- a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md +++ b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md @@ -9,7 +9,7 @@ relations: --- # Rebuild a per-entity read model online while commands run; interrupt and resume -Sc L2-5 · native tier · production composition. +Sc L2-5 · native backend tier · production composition. ## Intent @@ -29,7 +29,7 @@ And writes {writes: "never stopped"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test, in a file whose own timeout is 300 s, installs the order summary's generation 1 through its first rebuild, seeds 350 orders across two tenants, 250 in the first in tenant-ID order and 100 in the second, starts generation 2 with `projectionVersion` 1 stated, a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that `batchesDone` then stops advancing below 350 and that the cursor names a tenant; while a chain is expected to run, a progress row whose `batchesDone` has not advanced for 30 s fails the test with the generation row and the progress row in the message; once generation 2 is `verified` it asserts that the backfill and verify completions in the function log that wrote number the progress row's `batchesDone`, and that each pass took at least the 350 seeded orders and at most the orders at `verified` plus one per tenant. - While the chain is interrupted the test sends `CancelOrder` to one of the second tenant's 100 seeded orders and to an order of the first tenant whose row generation 2 already holds, read from the table before the command; it asserts at once that the first tenant's order has its row in generation 2 `cancelled`, and, once generation 2 is `verified`, that the second tenant's order has its row in generation 2 `cancelled`, whichever pass wrote it. - After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that, once the generation is `verified`, every order the Orders context's `list` enumerates in either tenant, read with admin access and `includeDeleted`, has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, that `backfillBatch` run with admin access under the fence the interrupt replaced leaves the progress row unchanged, that the function log shows no action and no completion during the batches other than a batch, an operator entry or a live command, and that the journals, read by operation, hold the event and receipt counts the live commands produced. diff --git a/design/specs/application/rebuild.write-pause-rebuild-abort.sdp.md b/design/specs/application/rebuild.write-pause-rebuild-abort.sdp.md index c4cb1e7..9e3c692 100644 --- a/design/specs/application/rebuild.write-pause-rebuild-abort.sdp.md +++ b/design/specs/application/rebuild.write-pause-rebuild-abort.sdp.md @@ -9,7 +9,7 @@ relations: --- # Rebuild a cross-stream history view under a write pause; interrupt, then abort -Sc L2-6 · native tier · the abort case, the doc's second enumerated case of the row. +Sc L2-6 · native backend tier · the abort case, the doc's second enumerated case of the row. ## Intent @@ -29,7 +29,7 @@ And writes {writes: "reopen after the abort"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The view is the order allocation history of `spec:application.orders-inventory-example`, a `HistoryProjection` bound to the order stream type and the stock item stream type. - The test starts a generation with `pauseRequired`, asserts that the gate holds one closed `source:` entry for each of the two stream types, each naming this generation, interrupts after the second history batch and calls `abortGeneration`. - The test asserts that the generation is `aborted`, that the active generation is unchanged, that both source entries are gone from the gate, that `PlaceOrder` succeeds again, and that no query returns a row of the aborted generation. diff --git a/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md b/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md index 19af9de..2572763 100644 --- a/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md +++ b/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md @@ -9,7 +9,7 @@ relations: --- # Rebuild a cross-stream history view under a write pause; interrupt, then resume -Sc L2-6 · native tier · the resume case. +Sc L2-6 · native backend tier · the resume case. ## Intent @@ -29,7 +29,7 @@ And writes {writes: "reopen only after verification"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The view is the order allocation history of `spec:application.orders-inventory-example`, a `HistoryProjection` bound to the order stream type and the stock item stream type. - The test starts a generation with `pauseRequired` and asserts that the gate holds one closed `source:` entry for each of the two stream types, each naming this generation. - The test asserts that `PlaceOrder` and `CancelOrder` are refused with `writePaused` because their declarations' `writes` name a closed source, that the error's `data.kind` is `"transient"`, and that nothing is stored. diff --git a/design/specs/application/restore.restore-representative-dataset.sdp.md b/design/specs/application/restore.restore-representative-dataset.sdp.md index 1b52507..caaa972 100644 --- a/design/specs/application/restore.restore-representative-dataset.sdp.md +++ b/design/specs/application/restore.restore-representative-dataset.sdp.md @@ -9,7 +9,7 @@ relations: --- # Restore a representative dataset with matching code and configuration -Sc L2-8 · end to end tier. +Sc L2-8 · native backend tier · production composition. ## Intent @@ -26,7 +26,7 @@ And a generation stranded in the snapshot is {stranded: "resumed by the accepted ## Verification — executable -- Runs in the end-to-end tier on a disposable deployment with the production composition; every test owns its disposable backend. +- Runs in the native backend tier on a disposable deployment with the production composition; every test owns its disposable backend. - The test builds the representative dataset with `PlaceOrder` and `CancelOrder` across two tenants, starts an online generation of the order summary and stops the scheduler after its first batch so the generation is in `building` at backup time, takes a backup, records the commit and hashes, then restores into a second disposable deployment with the same commit. - The test asserts that `PlaceOrder` is refused with `writePaused` while `MAINTENANCE_MODE` is `restore`, that all four checks reach `passed` with zero findings, that the pending-work inventory lists the building generation with its fence and no closed gate entry, that the Orders and Inventory components' tables came back with the parent's, which is the assumption the drill proves for Probe 7, and that `finishRestoreRun` refuses while the door is still set. - The test clears the door, calls `finishRestoreRun`, and asserts that the run record is `accepted` with the generation item marked `resumed`, that the generation's fence was bumped and its next batch scheduled, that it reaches `verified` covering both tenants' orders, that `PlaceOrder` succeeds, and that a receipt from before the backup is still recognized as a duplicate. diff --git a/design/specs/application/write-pause.restore-door-without-deploy.sdp.md b/design/specs/application/write-pause.restore-door-without-deploy.sdp.md index 0fda413..41afcee 100644 --- a/design/specs/application/write-pause.restore-door-without-deploy.sdp.md +++ b/design/specs/application/write-pause.restore-door-without-deploy.sdp.md @@ -9,7 +9,7 @@ relations: --- # The restore door closes and opens with no deploy -E-42 · native tier · fixture composition · no acceptance row; the example shows, on the pinned backend release, that a change of the deployment's environment reaches the next function run, which the restore door rests on. +E-42 · native backend tier · fixture composition · no acceptance row; the example shows, on the pinned backend release, that a change of the deployment's environment reaches the next function run, which the restore door rests on. ## Intent @@ -26,7 +26,7 @@ And once the variable is set to {reopen: "off"} each refused request key applies ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test sets the variable through the admin API's update of environment variables, with no deploy between the change and the next call, and asserts the refusal's data exactly, `{ kind: "transient", code: "writePaused", message }`, and that no receipt beyond the first command's exists while the door is closed. - The test also sets the value `Restore` and asserts that `getGate` answers `restore` false, because only the exact value closes the door. - The observation is of the pinned release and is read again when the pin moves. diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index c799b3a..fe49404 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -75,7 +75,7 @@ The gate is one document so that a command pays one point read. Scopes are strin - fnGetGate: `export const getGate = internalQuery({ args: {}, returns: v.object({ restore: v.boolean(), closed: v.array(closedEntryValidator) }), handler })` answering `restore` true when `MAINTENANCE_MODE` is `restore` and `closed` as the document holds it, empty when there is no document (E-8, E-42, D19) - fnGetGateAudit: `export const getGateAudit = internalQuery({ args: { scopeKey: v.string(), paginationOpts: paginationOptsValidator }, returns: paginationResultValidator(operatorAuditDoc), handler })` paging the `operatorAudit` records of one scope by `by_scope`, newest first, at most 100 a page, a request for more answered with a page of 100, passing the paginator only the request's `cursor` and the capped `numItems`, so an `endCursor` in the request is ignored and cannot widen a page, where `operatorAuditDoc` is the `operatorAudit` table's document validator with `_id` and `_creationTime` (E-8, E-43, D19) - occSerialization: a writer's mutation reads the gate document, or the index range that holds none; a close that commits first invalidates that read, the writer re-runs, reads the closed gate and refuses; a writer that commits first wrote before the close, which is the consistent cut; a batch scheduled before the close but started after it reads the closed gate like any writer, so no closing state, drain interval or confirmation is needed, and the one-second mutation timeout plays no part (F1, D9, E-8) -- observedRestoreDoorWithoutDeploy: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, a change of `MAINTENANCE_MODE` through the admin API's update of environment variables is seen by the next function run with no deploy: set to `restore`, the next command in every tenant, through the public and the internal entry, is refused with `"write paused for all: restore"` and `getGate` answers `restore` true; set to another value, `Restore` among them, the refused request keys apply; `spec:application.write-pause.restore-door-without-deploy` binds it at the native tier, and it is read again when the pin moves (E-42, F12) +- observedRestoreDoorWithoutDeploy: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, a change of `MAINTENANCE_MODE` through the admin API's update of environment variables is seen by the next function run with no deploy: set to `restore`, the next command in every tenant, through the public and the internal entry, is refused with `"write paused for all: restore"` and `getGate` answers `restore` true; set to another value, `Restore` among them, the refused request keys apply; `spec:application.write-pause.restore-door-without-deploy` binds it at the native backend tier, and it is read again when the pin moves (E-42, F12) - limitClosedScopes: at most 16 scopes closed at once, so the gate document stays a few kilobytes and the writer check scans an array in memory (E-8, F13) - limitGateReason: a reason is 1 to 256 bytes of UTF-8, so the refusal's message stays under a kilobyte (E-8, F13) - limitGateReadsPerCommand: 1 point read per command that reaches step 7, none for a duplicate answered from its receipt, and none while the restore door is closed; the first experiment counts it (E-8, Sc L2-3) diff --git a/design/specs/command/actor-and-scope.sdp.md b/design/specs/command/actor-and-scope.sdp.md index 49d8b3c..c106d49 100644 --- a/design/specs/command/actor-and-scope.sdp.md +++ b/design/specs/command/actor-and-scope.sdp.md @@ -73,7 +73,7 @@ This contract pins the shapes the parent establishes and passes into components: - fnAuthorize: `authorize(ctx: QueryCtx | MutationCtx, input: AuthorizeInput): Promise` reads `by_principal` and matches `permission` and, when the grant names a subject, the subject; a grant with no subject covers every subject, and one that names a subject covers that subject only and never an input with no subject; the reason is `subject_mismatch` when grants for the permission exist but none matches, and `no_grant` otherwise (D11, Law 5) - fnEstablishActor: `establishActor(ctx: MutationCtx | QueryCtx, serviceIssuers: ReadonlySet): Promise` returns `null` when `ctx.auth.getUserIdentity()` is `null`, a `service` actor when the identity's issuer is in `serviceIssuers`, and a `human` actor otherwise, with `id` from `tokenIdentifier` (D11, E-37) - typeQueryPolicy: `type QueryPolicy = { name: string; tenantId: string; permission: string; subject?: SubjectRef }` what a parent query states before it discloses anything: its own name, the tenant it reads and the permission the read requires (D11, Law 5, E-6) -- fnAuthorizeQuery: `authorizeQuery(ctx: GenericQueryCtx, policy: QueryPolicy, serviceIssuers?: ReadonlySet): Promise` calls `establishActor` and then `authorize` for the policy's tenant, permission and subject, as the pipeline's steps 2 and 4 do, and returns the actor; it throws through `reject` with code `unauthenticated` when there is no identity and `forbidden` with `details.reason` when the decision denies, each with the query's `name` where a command's type goes; it writes nothing, and it is generic over the parent's data model because a database reader typed with the parent's tables is not assignable to one typed with the library's (D11, Law 5, D7, E-6) +- fnAuthorizeQuery: `authorizeQuery(ctx: GenericQueryCtx, policy: QueryPolicy, serviceIssuers?: ReadonlySet): Promise` calls `establishActor` and then `authorize` for the policy's tenant, permission and subject, as the pipeline's steps 2 and 4 do, and returns the actor; it throws through `reject` with code `unauthenticated` when there is no identity and `forbidden` with `details.reason` when the decision denies, each with the query's `name` in `entry`; it writes nothing, and it is generic over the parent's data model because a database reader typed with the parent's tables is not assignable to one typed with the library's (D11, Law 5, D7, E-6) - fnInsertGrant: `insertGrant(ctx: { db: MutationCtx["db"] }, grant: GrantInput): Promise>` first refuses a `grant.tenantId` of more than `limitIdLength` bytes of UTF-8 with the rejection `invalidInput`, the `ConvexError` data `{ code: "invalidInput", message: "tenantId has at most 256 bytes of UTF-8", details: { field: "tenantId", length, limit: 256 } }`, before any read, then calls `assertWritable(ctx, scopesOfUseCase(grant.tenantId, []))` of `spec:application.write-pause`, so a closed `all` or tenant scope, or the restore door, refuses it with `writePaused` before it reads anything, then reads `tenants` by `by_tenant` for `grant.tenantId` and inserts `{ tenantId, createdAt }` with the mutation's `Date.now()` when no row exists, then inserts one grant row with `grantedAt` the mutation's `Date.now()`, where `GrantInput = { tenantId: string; principalKind: ActorKind; principalId: string; permission: string; subject?: SubjectRef; grantedBy: string }`; a plain helper the library exports and registers no function for (D11, E-37, E-49, E-8) - fnRevokeGrant: `revokeGrant(ctx: { db: MutationCtx["db"] }, grant: Omit): Promise` makes the gate read of `insertGrant` first, then deletes the rows that grant this permission to this principal, on the named subject when one is given and otherwise on any, and returns how many it deleted; revocation is that delete (D11, E-37, E-8) - grantWriters: the application registers its own internal mutations over `insertGrant` and `revokeGrant`; a tenant's first grant is created with admin access through that internal mutation, and later grants the same way or by whatever command the composition registers; the library registers no grant function and promises no recorded bootstrap, first-admin flow or invitation, which are an adopter's to build on the helpers; both helpers read the maintenance gate first, so the composition's grant mutations are held by a closed tenant, a closed deployment and the restore door like any writer; a native test creates its grants as setup with the harness's admin access (D11, E-37, E-15) diff --git a/design/specs/command/command-declaration.broken-argument-contract-fails-build.sdp.md b/design/specs/command/command-declaration.broken-argument-contract-fails-build.sdp.md index e176381..6877d64 100644 --- a/design/specs/command/command-declaration.broken-argument-contract-fails-build.sdp.md +++ b/design/specs/command/command-declaration.broken-argument-contract-fails-build.sdp.md @@ -9,7 +9,7 @@ relations: --- # Break a typed handler's argument contract -Sc L2-4 · build tier · the second of the row's two cases. +Sc L2-4 · compiled tier · the second of the row's two cases. A typed caller calls `api.ordering.placeOrder` with an input that no longer fits the declaration's `input` validator: a required field is missing, or a field has another type. The generated `api` type carries the argument type of the public entry, which the composition helper builds from the declaration, so the caller fails `tsc` before anything is deployed. @@ -27,6 +27,6 @@ And it fails {when: "before any traffic"} ## Verification — executable -- Runs in the build tier, the fifth tier `spec:platform.acceptance-contract` adds under E-14, with no backend, as a type test of the production composition's generated `api`. +- Runs in the compiled tier, the fifth tier `spec:platform.acceptance-contract` adds under E-14, with no backend, as a type test of the production composition's generated `api`. - The binding is a type test: the compiler is the check, and no step body runs. - The test asserts that a call of `api.ordering.placeOrder` whose input lacks a required field, and one whose field has another type, each fail `tsc` through the compiler's expected-error directive. diff --git a/design/specs/command/command-declaration.renamed-handler-fails-build.sdp.md b/design/specs/command/command-declaration.renamed-handler-fails-build.sdp.md index a90983a..ef21452 100644 --- a/design/specs/command/command-declaration.renamed-handler-fails-build.sdp.md +++ b/design/specs/command/command-declaration.renamed-handler-fails-build.sdp.md @@ -9,7 +9,7 @@ relations: --- # Rename a typed handler -Sc L2-4 · build tier · the first of the row's two cases. +Sc L2-4 · compiled tier · the first of the row's two cases. The export `placeOrder` is renamed to `submitOrder` without touching its callers. The generated `api` type is built from the module's exports, so `api.ordering.placeOrder` no longer exists and every typed caller fails `tsc`. @@ -27,6 +27,6 @@ And it fails {when: "before any traffic"} ## Verification — executable -- Runs in the build tier, the fifth tier `spec:platform.acceptance-contract` adds under E-14, with no backend, as a type test of the production composition's generated `api`. +- Runs in the compiled tier, the fifth tier `spec:platform.acceptance-contract` adds under E-14, with no backend, as a type test of the production composition's generated `api`. - The binding is a type test: the compiler is the check, and no step body runs. - The test asserts that `api.ordering.placeOrder` is a public mutation reference, that a reference to a name the module does not export fails `tsc`, and that a typed call of `placeOrder` through the api built, as the generated `api` is, from the module with that export renamed `submitOrder` fails `tsc`, each through the compiler's expected-error directive, which itself fails the compile when the line stops being an error. diff --git a/design/specs/command/command-pipeline.failure-after-journal-append.sdp.md b/design/specs/command/command-pipeline.failure-after-journal-append.sdp.md index 1c3a549..5e22bc9 100644 --- a/design/specs/command/command-pipeline.failure-after-journal-append.sdp.md +++ b/design/specs/command/command-pipeline.failure-after-journal-append.sdp.md @@ -9,7 +9,7 @@ relations: --- # Inject a failure after the journal append -Sc L1-2 · native tier · fixture composition · second of three injection points. +Sc L1-2 · native backend tier · fixture composition · second of three injection points. The fixture context `depot` registers its document stream with a mapping whose `isDeleted` throws a plain Error for a document whose title is the fixture's after-journal-append fault title. The adapter's `execute` calls `isDeleted` at its step 9, after `append` inserted the new event and before the stream row is written, so an ordinary client with a grant sends a receipted `AmendDocument` with that title to a document at version 3. The append and the state save sit in one sub-transaction, so a throw after the append rolls back the events together with the state. @@ -29,7 +29,7 @@ And an event from the command exists in the journal {eventExists: false} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test reads the depot's events table after the failure and asserts that the stream holds exactly the three events of its setup commands, each under an operation ID a setup response returned, because the failed command's operation ID never reaches the caller. - The test asserts that the stream's metadata still names version 3, so a later command with expected version 3 succeeds. - The test asserts that the function log holds one completion record for the request, the parent mutation's own, carrying the fault's error. diff --git a/design/specs/command/command-pipeline.failure-after-state-write.sdp.md b/design/specs/command/command-pipeline.failure-after-state-write.sdp.md index eed245e..063eb53 100644 --- a/design/specs/command/command-pipeline.failure-after-state-write.sdp.md +++ b/design/specs/command/command-pipeline.failure-after-state-write.sdp.md @@ -9,7 +9,7 @@ relations: --- # Inject a failure after the state write -Sc L1-2 · native tier · fixture composition · first of three injection points. +Sc L1-2 · native backend tier · fixture composition · first of three injection points. The fixture context `depot` registers its document stream with a `toDto` that throws a plain Error for a document whose title is the fixture's after-state-write fault title. The adapter's `execute` calls `toDto` at its step 10, after it appended the event and saved the folded state, so an ordinary client with a grant sends a receipted `AmendDocument` with that title to a document at version 3. The throw happens inside the component's sub-transaction, so the sub-transaction rolls back and, uncaught by the parent, the whole mutation with it. @@ -29,7 +29,7 @@ And an event from the command exists in the journal {eventExists: false} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that the context's state document is byte-identical to its pre-command value, not only that the version is unchanged. - The test asserts that the thrown error is not a `ConvexError`, so the client cannot mistake the technical failure for a rejection. - The test asserts that the function log holds one completion record for the request, the parent mutation's own, carrying the fault's error. diff --git a/design/specs/command/command-pipeline.failure-before-receipt.sdp.md b/design/specs/command/command-pipeline.failure-before-receipt.sdp.md index 2d8544b..3b81743 100644 --- a/design/specs/command/command-pipeline.failure-before-receipt.sdp.md +++ b/design/specs/command/command-pipeline.failure-before-receipt.sdp.md @@ -9,7 +9,7 @@ relations: --- # Inject a failure before the receipt insert -Sc L1-2 · native tier · fixture composition · third of three injection points. +Sc L1-2 · native backend tier · fixture composition · third of three injection points. The fault sits in the parent, in the fixture composition's executor for `AmendDocument`: after the `depot` context's sub-transaction has returned its DTO and versions, the executor throws a plain Error that names the returned versions while the command's `failBeforeReceipt` switch, a fixture table row the test sets with admin access, is on. The throw ends step 8, so neither step 9 nor the receipt insert of step 10 runs. An ordinary client with a grant sends the receipted `AmendDocument` to a document at version 3. This point proves that a component's sub-transaction does not commit on its own: the context call completed without error, and still nothing of it survives the parent's throw. @@ -29,7 +29,7 @@ And an event from the command exists in the journal {eventExists: false} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that the thrown error names version 4 of the document, which only the completed context call returned, and that the context's state document is byte-identical to its pre-command value. - The test asserts that the function log holds one completion record for the request, the parent mutation's own, carrying the fault's error. - The test then turns the switch off, sends the same command with the same request key and asserts that it executes as new intent and returns `replayed` false, because no receipt survived. diff --git a/design/specs/command/idempotency-and-receipts.capacity-refusal-then-retry.sdp.md b/design/specs/command/idempotency-and-receipts.capacity-refusal-then-retry.sdp.md index b7d0898..f04f8b3 100644 --- a/design/specs/command/idempotency-and-receipts.capacity-refusal-then-retry.sdp.md +++ b/design/specs/command/idempotency-and-receipts.capacity-refusal-then-retry.sdp.md @@ -9,7 +9,7 @@ relations: --- # Capacity refusal, then a retry of the same intent -Sc L1-9 · native tier · fixture composition · second of two cases. +Sc L1-9 · native backend tier · fixture composition · second of two cases. A worker sends the depot's `AddStock` command through the fixture action `nonUiCaller:send` that an ordinary client calls; the input f-1 adds 5 units of product p-1. The command's admission policy, the fixture's `switchedAdmission`, refuses the first call as a transient `capacity` while the command's `capacity` switch is on. The refusal throws, so nothing is stored and no receipt exists. The switch is turned off with admin access, the caller retries with the same key and input, admission now admits, and the command executes as new intent. @@ -32,7 +32,7 @@ And the original outcome and state are unchanged {unchanged: true} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that the first error has `data.kind` equal to `"transient"`, `data.code` equal to `"capacity"` and no `data.retryAfterMs`, and that the receipts table and the depot's tables are empty after it. - The test asserts that the retry's response has `replayed` false, which proves the refusal left no receipt to replay. - The test also asserts the ordering rule: with a receipt already present for the key, a refusing admission policy is never consulted and the duplicate is replayed; the switch is turned on again, the same call is replayed and stores nothing, and a call under a new request key is refused, which shows on the same deployment that the policy refuses whenever it is consulted. diff --git a/design/specs/command/idempotency-and-receipts.concurrent-non-ui-calls.sdp.md b/design/specs/command/idempotency-and-receipts.concurrent-non-ui-calls.sdp.md index 2aaedce..449b87d 100644 --- a/design/specs/command/idempotency-and-receipts.concurrent-non-ui-calls.sdp.md +++ b/design/specs/command/idempotency-and-receipts.concurrent-non-ui-calls.sdp.md @@ -9,7 +9,7 @@ relations: --- # A non-UI caller sends the same command concurrently -Sc L1-3 · native tier · fixture composition · first of two cases. +Sc L1-3 · native backend tier · fixture composition · first of two cases. A worker holding a grant for `depot.stock` calls the internal entry of the depot's `AddStock` command twice at once with the same request key and input, through the fixture action `nonUiCaller:send` that an ordinary client calls; the input f-1 adds 5 units of product p-1. Both mutations read the empty key range; one commits its insert, the other's read set is invalidated, the engine retries it, and the retry finds the receipt and answers as a duplicate. @@ -32,7 +32,7 @@ And the original outcome and state are unchanged {unchanged: true} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that both responses carry the same `operationId` and that the journal holds exactly one event set with that operation ID. - The test asserts that which call is answered as the duplicate is not fixed; it checks the pair, not the order. - The test asserts from the function log that one of the two internal-entry mutations failed its commit on a conflict the engine reports as an OCC conflict with a retry to follow, and that its rerun in the same request completed without error. diff --git a/design/specs/command/idempotency-and-receipts.key-reuse-changed-input.sdp.md b/design/specs/command/idempotency-and-receipts.key-reuse-changed-input.sdp.md index 4119d04..6154a34 100644 --- a/design/specs/command/idempotency-and-receipts.key-reuse-changed-input.sdp.md +++ b/design/specs/command/idempotency-and-receipts.key-reuse-changed-input.sdp.md @@ -9,7 +9,7 @@ relations: --- # Reuse a key with changed business input -Sc L1-5 · native tier · fixture composition. +Sc L1-5 · native backend tier · fixture composition. A service caller sends the depot's `AddStock` command through the fixture action `nonUiCaller:send` that an ordinary client calls. A receipt for the key already exists with the fingerprint of the original input f-1, which added 5 units of product p-1. The caller sends the same key with different business input f-2, which adds 7 units. The lookup finds a different fingerprint and throws `idempotencyConflict` before any execution. @@ -32,7 +32,7 @@ And the original outcome and state are unchanged {unchanged: true} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that the thrown error has `data.kind` equal to `"rejection"`, `data.code` equal to `"idempotencyConflict"` and no `details`, and that the original receipt's operation ID appears nowhere in the error's data. - The test asserts that the receipt row is byte-identical before and after, and that the stream version the original touched is unchanged. - The test asserts that the stored fingerprint is `fingerprintOf` of the original input and differs from that of the changed input, so f-1 and f-2 name two real fingerprints. diff --git a/design/specs/command/idempotency-and-receipts.rate-refusal-then-retry.sdp.md b/design/specs/command/idempotency-and-receipts.rate-refusal-then-retry.sdp.md index 37472a5..d3e13f3 100644 --- a/design/specs/command/idempotency-and-receipts.rate-refusal-then-retry.sdp.md +++ b/design/specs/command/idempotency-and-receipts.rate-refusal-then-retry.sdp.md @@ -9,7 +9,7 @@ relations: --- # Rate refusal, then a retry of the same intent -Sc L1-9 · native tier · fixture composition · first of two cases. +Sc L1-9 · native backend tier · fixture composition · first of two cases. A worker sends the depot's `AddStock` command through the fixture action `nonUiCaller:send` that an ordinary client calls; the input f-1 adds 5 units of product p-1. The command's admission policy, the fixture's `switchedAdmission`, refuses the first call as a transient `rateLimited` while the command's `rateLimited` switch is on. The refusal throws, so nothing is stored and no receipt exists. The switch is turned off with admin access, the caller retries with the same key and input, admission now admits, and the command executes as new intent. @@ -32,7 +32,7 @@ And the original outcome and state are unchanged {unchanged: true} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that the first error has `data.kind` equal to `"transient"`, `data.code` equal to `"rateLimited"` and `data.retryAfterMs` equal to 1000, and that the receipts table and the depot's tables are empty after it. - The test asserts that the retry's response has `replayed` false, which proves the refusal left no receipt to replay. - The test also asserts the ordering rule: with a receipt already present for the key, a refusing admission policy is never consulted and the duplicate is replayed; the switch is turned on again, the same call is replayed and stores nothing, and a call under a new request key is refused, which shows on the same deployment that the policy refuses whenever it is consulted. diff --git a/design/specs/command/idempotency-and-receipts.retry-after-lost-response.sdp.md b/design/specs/command/idempotency-and-receipts.retry-after-lost-response.sdp.md index 09140f8..90d7f7d 100644 --- a/design/specs/command/idempotency-and-receipts.retry-after-lost-response.sdp.md +++ b/design/specs/command/idempotency-and-receipts.retry-after-lost-response.sdp.md @@ -9,7 +9,7 @@ relations: --- # A non-UI caller sends the same command again after a lost response -Sc L1-3 · native tier · fixture composition · second of two cases. +Sc L1-3 · native backend tier · fixture composition · second of two cases. A worker calls the internal entry of the depot's `AddStock` command through the fixture action `nonUiCaller:send` that an ordinary client calls, the harness's transport loses the response after the backend answered, and the worker retries with the same request key and input; the input f-1 adds 5 units of product p-1. The retry finds the receipt with the same fingerprint and answers as a duplicate without executing. @@ -32,6 +32,6 @@ And the original outcome and state are unchanged {unchanged: true} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that the retry's response has `replayed` true, `result` null, and the same `operationId`, affected IDs and versions as the receipt row, and that it arrived as a return value, which proves the entry's `returns` validator admitted the replayed member rather than throwing on the null result. - The test asserts that the context operation was not called a second time: the events with the operation ID still number one, and the retry is sent while the fixture's `failBeforeReceipt` switch for `AddStock` is on, which fails the call once the command's executor has made its context call, so a replayed answer shows that neither ran; with the switch still on, the same input under a new request key fails as a technical failure and stores nothing, which shows on the same deployment that the switch fails a call that executes. diff --git a/design/specs/command/idempotency-and-receipts.ui-double-submit.sdp.md b/design/specs/command/idempotency-and-receipts.ui-double-submit.sdp.md index f47f6e4..1d57f62 100644 --- a/design/specs/command/idempotency-and-receipts.ui-double-submit.sdp.md +++ b/design/specs/command/idempotency-and-receipts.ui-double-submit.sdp.md @@ -9,7 +9,7 @@ relations: --- # The UI submits the same create twice -Sc L1-4 · native tier · fixture composition. +Sc L1-4 · native backend tier · fixture composition. A Convex client calls the public entry of the fixture command `PlaceOrder` twice, without a request key, with the same client-generated entity ID. `PlaceOrder` creates the order as a depot document under that ID and then claims the order's stock lines, in that order. No receipt is involved. The create is planned at expected version 0, so the persistence adapter's identity read answers the second call with the reserved rejection `entityExists` naming the first entity, before `decide` runs and inside the context's sub-transaction. @@ -31,7 +31,7 @@ And the original outcome and state are unchanged {unchanged: true} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that exactly one stream with `streamId` equal to the entity ID exists in the tenant, that the second call's `error.data.code` is `entityExists`, and that `error.data.details.existing` equals the entity ID. - The test asserts that the document decider's `decide` did not run for the second call, which shows the adapter's expected-version read is the uniqueness check: the code is `entityExists` and not the `invalidTransition` that `decide` answers for a create of an existing document, and the fixture command `FailIfDecided`, whose `decide` fails the call if it is reached, sent for the same document at expected version 0 is answered `entityExists`, while sent at the document's current version 1 it fails as a technical failure, which shows on the same deployment that its `decide` runs when reached. - The test asserts that the receipts table stays empty, which proves the UI path wrote no receipt. diff --git a/design/specs/command/outcome-boundary.rejected-then-retried-after-change.sdp.md b/design/specs/command/outcome-boundary.rejected-then-retried-after-change.sdp.md index 0ffdab6..24d1b70 100644 --- a/design/specs/command/outcome-boundary.rejected-then-retried-after-change.sdp.md +++ b/design/specs/command/outcome-boundary.rejected-then-retried-after-change.sdp.md @@ -9,7 +9,7 @@ relations: --- # A rejected command's response is lost, and it is retried after state changed -Sc L1-12 · native tier · fixture composition. +Sc L1-12 · native backend tier · fixture composition. On the fixture composition, the depot's stock stream for product `p-1` in tenant `t-1` stands at version 5 with nothing on hand, and an ordinary client that holds a stock grant sends the receipted `ClaimStock` for one unit. The first run is rejected because the stream's state refuses the command, an allocation against empty stock. The test keeps the thrown error for its assertions and otherwise acts as the caller whose response was lost. Another command then changes the stream, `AddStock` of one unit, a restock. The retry carries the same request key and input and must run against the new state, because the first attempt stored nothing, not even its key. @@ -28,7 +28,7 @@ And a receipt or record of the first attempt exists {firstAttemptRecorded: false ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts that the first run's thrown error has `data.kind` equal to `"rejection"` and `data.code` equal to `"insufficientStock"`, a domain code the `ClaimStock` declaration names. - The test asserts that the receipts table holds no row for the key after the first run, and exactly one row with `replayed` false in the retry's response after the second. - The test asserts that the retry's returned stream version is 7, one past the restock, which proves it decided against version 6 and not against the state the first attempt saw. diff --git a/design/specs/command/outcome-boundary.sdp.md b/design/specs/command/outcome-boundary.sdp.md index 6355c35..79633b7 100644 --- a/design/specs/command/outcome-boundary.sdp.md +++ b/design/specs/command/outcome-boundary.sdp.md @@ -23,7 +23,7 @@ Layer 1 · Detail: full · Traces: D4, D7, Law 6, F3, F14, OQ2, Probe 2, Sc L1-1 This contract pins how the kernel's four outcomes cross the wire at the public boundary. Applied and business failure are returned as values and both commit. A rejection is thrown as a `ConvexError` whose data carries a closed code, and the mutation does not commit. A technical failure is any other throw; everything rolls back and nothing is stored as a business outcome. A transient refusal is a `ConvexError` of a second, retryable kind, and nothing is stored either. The `Outcome` union itself is defined once, in `spec:kernel.outcome-model`; this contract maps it and adds nothing to it. -The closed code list has two halves. The platform codes below are closed here and include the three codes the kernel reserves for the persistence adapter, `staleVersion`, `operationTooLarge` and `entityExists`. Domain codes, the documented rejections a context's `decide` can return such as `invalidTransition`, are the context's own and are closed per command by its declaration's `rejections` field, so the union a client sees is closed at build time and no code reaches the wire that no declaration names. Every code is lower camel case, as the kernel writes its reserved ones. A context throws the kernel's bare `Rejection` as `ConvexError` data; the boundary's one catch, at the outermost level of the pipeline's handler, rethrows it in the wire shape below with the discriminator and the command type added when its code is a platform code or one the declaration's `rejections` names, rethrows it as a plain error otherwise, and never continues past it. A rejection that reaches the catch already in the wire shape passes the same checks, of its code, its command name and the size of its details, or becomes a plain error too. The generic internal dispatcher that D7 allows for a boundary which must record a refusal is carried as a conditional design, gated on the doc's open question about refused-command records. +The closed code list has two halves. The platform codes below are closed here and include the three codes the kernel reserves for the persistence adapter, `staleVersion`, `operationTooLarge` and `entityExists`. Domain codes, the documented rejections a context's `decide` can return such as `invalidTransition`, are the context's own and are closed per command by its declaration's `rejections` field, so the union a client sees is closed at build time and no code reaches the wire that no declaration names. Every code is lower camel case, as the kernel writes its reserved ones. A context throws the kernel's bare `Rejection` as `ConvexError` data; the boundary's one catch, at the outermost level of the pipeline's handler, rethrows it in the wire shape below with the discriminator and the entry added when its code is a platform code or one the declaration's `rejections` names, rethrows it as a plain error otherwise, and never continues past it. A rejection that reaches the catch already in the wire shape passes the same checks, of its code, the entry it names and the size of its details, or becomes a plain error too. The generic internal dispatcher that D7 allows for a boundary which must record a refusal is carried as a conditional design, gated on the doc's open question about refused-command records. ## Intent diff --git a/design/specs/command/tenancy-and-authority.client-claims-agent-namespace.sdp.md b/design/specs/command/tenancy-and-authority.client-claims-agent-namespace.sdp.md index 45529ab..b9cd33d 100644 --- a/design/specs/command/tenancy-and-authority.client-claims-agent-namespace.sdp.md +++ b/design/specs/command/tenancy-and-authority.client-claims-agent-namespace.sdp.md @@ -9,7 +9,7 @@ relations: --- # A client claims an agent namespace -Sc L1-7 · native tier · fixture composition · second of two cases. +Sc L1-7 · native backend tier · fixture composition · second of two cases. On the fixture composition, an ordinary client that carries a fixture-issuer token for subject `user-1` and holds a grant in tenant `t-1` calls the internal entry of `CreateDocument` directly, naming the `agent` namespace and an actor of its own. The internal entry is an internal function, so Convex refuses a client's call to it before any handler runs. The other route to the agent namespace, a `namespace` argument to the public entry, fails the public entry's `args` validators and is asserted in the verification bullets. The worker namespace is the first case, `spec:command.tenancy-and-authority.client-claims-system-namespace`. @@ -29,7 +29,7 @@ And the namespace the server assigned is {namespace: "none"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts that the error names the internal entry as a function with no public form, that it is not a `ConvexError` of the outcome boundary, and that no receipt, stream or event row was written. - The test reads the function log and asserts that the refused call's completion record read no document, and that the same internal entry reached with namespace `agent` through the fixture's non-UI caller action `nonUiCaller:send` applies and stores a receipt whose namespace is `agent`, so the internal entry exists and the refusal came from Convex's visibility rule. - The test also sends the public entry a `namespace` argument of `agent` from the same client and asserts that the call fails argument validation, that its completion record read no document and that nothing was stored, so both routes to the agent namespace are closed to a client. diff --git a/design/specs/command/tenancy-and-authority.client-claims-system-namespace.sdp.md b/design/specs/command/tenancy-and-authority.client-claims-system-namespace.sdp.md index bbb626b..1387e71 100644 --- a/design/specs/command/tenancy-and-authority.client-claims-system-namespace.sdp.md +++ b/design/specs/command/tenancy-and-authority.client-claims-system-namespace.sdp.md @@ -9,7 +9,7 @@ relations: --- # A client claims a worker or agent namespace -Sc L1-7 · native tier · fixture composition · first of two cases. +Sc L1-7 · native backend tier · fixture composition · first of two cases. On the fixture composition, an ordinary client that carries a fixture-issuer token for subject `user-1` and holds a grant in tenant `t-1` sends the public entry of `CreateDocument` a `namespace` argument of `worker`. The public entry's `args` validators do not declare that field, so the call fails argument validation before the handler runs. The other route to the worker namespace, a client calling the internal entry directly, is refused by Convex's visibility rule and is asserted in the verification bullets. The agent namespace is the second case, `spec:command.tenancy-and-authority.client-claims-agent-namespace`. @@ -29,7 +29,7 @@ And the namespace the server assigned is {namespace: "none"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts that no receipt, stream or event row was written and that the error is not a `ConvexError` of the outcome boundary. - The test reads the function log and asserts that the refused call's completion record carries the argument validation error and read no document, and that the same call without the `namespace` field, sent afterwards by the same client, applies with namespace `public` and its completion record read the caller's grant, so the public entry exists and the refusal came before its handler. - The test also calls the internal entry from the same ordinary client, which holds no admin key, with namespace `worker` and asserts that Convex refuses the call, that its completion record read no document and that nothing was stored, then reaches the same internal entry with namespace `worker` through the fixture's non-UI caller action `nonUiCaller:send`, which applies, so the internal entry exists and both routes to the worker namespace are closed to a client. diff --git a/design/specs/command/tenancy-and-authority.revoked-then-retried.sdp.md b/design/specs/command/tenancy-and-authority.revoked-then-retried.sdp.md index 94f9966..865d785 100644 --- a/design/specs/command/tenancy-and-authority.revoked-then-retried.sdp.md +++ b/design/specs/command/tenancy-and-authority.revoked-then-retried.sdp.md @@ -9,7 +9,7 @@ relations: --- # Authorization is revoked, then a successful command is retried -Sc L1-8 · native tier · fixture composition. +Sc L1-8 · native backend tier · fixture composition. On the fixture composition, an ordinary client that carries a fixture-issuer token for subject `user-1` sends the receipted `CreateDocument` through its public entry, which succeeds and leaves a receipt. An operator then deletes the caller's grant with the fixture's internal mutation `grants:revoke`, which the test runs with admin access. The caller retries with the same key and input. Authorization runs at step 4, before the receipt lookup of step 5, so the retry receives `forbidden` and the stored outcome is not disclosed. The fixture configures no service issuer, so the public entry makes the caller a human actor. @@ -28,7 +28,7 @@ And the namespace the server assigned is {namespace: "public"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts that the thrown error has `data.code` equal to `"forbidden"` and that `data.details` carries no operation ID, affected ref or version. - The test asserts that the receipt row from the successful run is unchanged, which proves the retry wrote nothing, and that the retry's completion record in the function log read no document, which proves the retry never read the receipt it could disclose. - After the test grants the caller again with admin access, the same retry is a replay that names the stored operation ID, so the receipt was there to disclose and the `forbidden` answer came from authorization. diff --git a/design/specs/command/tenancy-and-authority.same-key-two-tenants.sdp.md b/design/specs/command/tenancy-and-authority.same-key-two-tenants.sdp.md index 605241a..73be159 100644 --- a/design/specs/command/tenancy-and-authority.same-key-two-tenants.sdp.md +++ b/design/specs/command/tenancy-and-authority.same-key-two-tenants.sdp.md @@ -9,7 +9,7 @@ relations: --- # Two tenants use the same request key and local ID -Sc L1-6 · native tier · fixture composition. +Sc L1-6 · native backend tier · fixture composition. On the fixture composition, tenant A's own caller has already applied the receipted create `CreateDocument` through its public entry with request key `k-1` and document ID `doc-1`. Tenant B's caller, the fixture issuer's subject `user-b`, sends the same command with the same key, document ID and title to tenant B. The receipt key leads with the tenant, so the lookup finds nothing; the document ID is scoped by the tenant in the depot's identity index, so the create succeeds; and nothing of tenant A is read or returned. @@ -29,6 +29,6 @@ And the namespace the server assigned is {namespace: "public"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test asserts that two receipt rows exist, one per tenant, with different operation IDs, and two rows in the depot's streams table with `streamId` `doc-1`, one per tenant. - The test asserts that tenant B's response carries no operation ID or version from tenant A: its operation ID is the one tenant B's receipt stores, and every version in it names tenant `t-b`; an affected ref names no tenant, so the test asserts it equals the ref tenant B's receipt stores. diff --git a/design/specs/command/tenancy-and-authority.sdp.md b/design/specs/command/tenancy-and-authority.sdp.md index 1ce4969..a6aade7 100644 --- a/design/specs/command/tenancy-and-authority.sdp.md +++ b/design/specs/command/tenancy-and-authority.sdp.md @@ -75,7 +75,7 @@ Nothing here signs anything. The internal entry is unreachable by clients becaus - clientClaimRefused: a `namespace` or `actor` field sent to the public entry fails its `args` validator, and a client call to the internal entry is refused by Convex before any handler runs (D11, Sc L1-7) - workerModes: the two modes of the one `Authority` shape pinned in `spec:command.actor-and-scope`, never restated here: `recheckDelegator` re-reads the delegating user's grants at execution through `authorize` with the captured actor and scope, and `serviceAuthority` runs under the service actor's own grants with no recheck; both carry the obligation on the actor's `delegationRef` (D11, D13, E-6) - trustBoundary: one trusted deployment; component privacy and type brands, never a signed token, and no claim against a malicious administrator (D11) -- nativeTierIdentity: on the native tier the identity `ctx.auth.getUserIdentity()` returns comes from a token the harness signs for the fixture issuer that the deployment's environment variables name, carried by an ordinary client, so `establishActor`, the grants read, the namespaces and Convex's visibility check run unchanged; the issuer is the identity source of a native run and its one named difference from production (E-13, S13, Sc L2-9) +- nativeTierIdentity: on the native backend tier the identity `ctx.auth.getUserIdentity()` returns comes from a token the harness signs for the fixture issuer that the deployment's environment variables name, carried by an ordinary client, so `establishActor`, the grants read, the namespaces and Convex's visibility check run unchanged; the issuer is the identity source of a native run and its one named difference from production (E-13, S13, Sc L2-9) - limitGrantsPerPrincipal: at most 500 grants read per principal per tenant per command; the read takes 501 and a 501st row throws a plain `Error`, a technical failure, never a silent truncation, because `.take(500)` alone cannot show that a 501st exists; a principal with more is a product problem the failure surfaces (F13, E-37) ## Example space diff --git a/design/specs/command/tenancy-and-authority.shared-key-two-subjects.sdp.md b/design/specs/command/tenancy-and-authority.shared-key-two-subjects.sdp.md index 1a5c4e9..ef32d17 100644 --- a/design/specs/command/tenancy-and-authority.shared-key-two-subjects.sdp.md +++ b/design/specs/command/tenancy-and-authority.shared-key-two-subjects.sdp.md @@ -9,7 +9,7 @@ relations: --- # A request key taken for another subject is reused with other input -Law 5 · native tier · fixture composition. +Law 5 · native backend tier · fixture composition. On the fixture composition, two ordinary clients carry fixture-issuer tokens for the subjects `user-1` and `user-2`. Each holds one grant for the permission of `CreateDocument`, and each grant names one document as its subject: `doc-1` for `user-1` and `doc-2` for `user-2`. The client of `user-2` sends the receipted `CreateDocument` for `doc-2` through its public entry with request key `k-1`, which succeeds and leaves a receipt. The client of `user-1` then sends `CreateDocument` for `doc-1` with the same key. Authorization passes at step 4, because the grant covers `doc-1`. The lookup of step 5 finds the receipt of the other call with a different fingerprint and throws `idempotencyConflict` with no details, so the caller learns that the key is taken and nothing of the receipt recorded for `doc-2` except the one fact its code cannot hide, that the receipt's contract version is the declaration's, because `unsupportedContractVersion` answers otherwise. @@ -30,7 +30,7 @@ And the namespace the server assigned is {namespace: "public"} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. -- The test asserts that the thrown error has `data.kind` equal to `"rejection"`, `data.code` equal to `"idempotencyConflict"`, `data.commandType` equal to `"CreateDocument"` and no `details`, and that the operation ID of the stored receipt appears nowhere in the error's data. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. +- The test asserts that the thrown error has `data.kind` equal to `"rejection"`, `data.code` equal to `"idempotencyConflict"`, `data.entry` equal to `"CreateDocument"` and no `details`, and that the operation ID of the stored receipt appears nowhere in the error's data. - The test asserts that the caller, sending `CreateDocument` for `doc-2` with a request key no call has used, is answered `forbidden`, so the stored receipt's subject is one the caller holds no grant for. - The test asserts that the receipt row is unchanged, that the receipts stored for the key number one, and that no stream exists for `doc-1`. diff --git a/design/specs/context/context-component.competing-commands.sdp.md b/design/specs/context/context-component.competing-commands.sdp.md index d2d69e3..134f5cc 100644 --- a/design/specs/context/context-component.competing-commands.sdp.md +++ b/design/specs/context/context-component.competing-commands.sdp.md @@ -9,7 +9,7 @@ relations: --- # Two commands compete for the same stock -Sc L1-11 · native tier · fixture composition · first of two cases. +Sc L1-11 · native backend tier · fixture composition · first of two cases. The fixture composition binds it. An ordinary client stocks one unit of a product in the depot context. Two other ordinary clients then send the ClaimStock command for that unit through its public entry at once, and the depot's claimStock operation plans a claim on the product's stock stream with no expected version. Both calls load version 1; one commits, the engine reruns the other, and the rerun decides against the stock the winner left. The unique-value case is the second. @@ -32,7 +32,7 @@ And the callers saw {saw: "no engine retry and no version conflict"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend, and the two parent mutations are issued at once from two ordinary clients, each holding a fixture-issuer token and a grant an operator gave with admin access, so that the backend runs them concurrently. +- Runs in the native backend tier; every test owns its disposable backend, and the two parent mutations are issued at once from two ordinary clients, each holding a fixture-issuer token and a grant an operator gave with admin access, so that the backend runs them concurrently. - The first and second callers are the callers in the order their final attempts complete in the function log. - Exactly one `claimed` event exists at version 2 and the stock reads 0; the loser's `ConvexError` carries the context's `insufficientStock` code with `details.onHand` of 0, the stock the winner left, and not `staleVersion`; neither caller received any error other than that rejection, which is what makes the engine retry invisible. - The test repeats the race on a fresh product, at most 20 times, until the function log shows an attempt of the parent mutation that the engine reruns after an OCC conflict, and every round meets the bullets above. diff --git a/design/specs/context/context-component.competing-unique-value.sdp.md b/design/specs/context/context-component.competing-unique-value.sdp.md index fd36f6f..7a3962d 100644 --- a/design/specs/context/context-component.competing-unique-value.sdp.md +++ b/design/specs/context/context-component.competing-unique-value.sdp.md @@ -9,7 +9,7 @@ relations: --- # Two commands compete for the same unique value -Sc L1-11 · native tier · fixture composition · second of two cases. +Sc L1-11 · native backend tier · fixture composition · second of two cases. The fixture composition binds it. Two ordinary clients send the RegisterDocument command at once, each creating its own document under one reference, the unique value. The depot's registerDocuments operation claims the reference on a reference stream of its own at expected version 0 and then creates the document, both in one sub-transaction. Both calls read the absent reference row; one commits, the engine reruns the other, and the rerun loads the claimed row and is answered by the version check before decide. @@ -32,7 +32,7 @@ And the callers saw {saw: "no engine retry and no version conflict"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend, and two ordinary clients, each holding a fixture-issuer token and a grant an operator gave with admin access, send RegisterDocument through its public entry at once with one reference and two document IDs. +- Runs in the native backend tier; every test owns its disposable backend, and two ordinary clients, each holding a fixture-issuer token and a grant an operator gave with admin access, send RegisterDocument through its public entry at once with one reference and two document IDs. - The first and second callers are the callers in the order their final attempts complete in the function log; the stream of the Given and of the version afterwards is the reference stream, and the events appended are the reference stream's `referenceClaimed` and the winner's document stream's `created`, both at version 1. - The reference stream's state names the winner's document as holder, and the loser's document has no stream row and no event, so no document exists without its claim. - The loser's `ConvexError` carries `entityExists` with `details.existing` equal to the reference and `details.current` equal to 1, the answer step 3 gives from the row the rerun loaded before `decide` runs; the reference decider would answer a held reference with `referenceTaken`, and that code is absent. diff --git a/design/specs/context/context-component.invalid-transition.sdp.md b/design/specs/context/context-component.invalid-transition.sdp.md index 5e100d3..62cd654 100644 --- a/design/specs/context/context-component.invalid-transition.sdp.md +++ b/design/specs/context/context-component.invalid-transition.sdp.md @@ -9,7 +9,7 @@ relations: --- # A command asks for an invalid state transition -Sc L1-1 · native tier · fixture composition. +Sc L1-1 · native backend tier · fixture composition. The fixture composition binds it. An ordinary client creates a document in the depot context and then sends the ShipDocument command through its public entry; the parent mutation calls the depot's shipDocuments operation through `ctx.runMutation` on the component API, and the document stream's transition table has no ship from draft. @@ -31,6 +31,6 @@ And decide ran against {evaluated: "the fresh state"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend; the caller is an ordinary client holding a fixture-issuer token and a grant, and admin access only grants and reads the depot's tables and the function log. +- Runs in the native backend tier; every test owns its disposable backend; the caller is an ordinary client holding a fixture-issuer token and a grant, and admin access only grants and reads the depot's tables and the function log. - The rejection arrives as a `ConvexError` whose `data.code` is the context's documented `invalidTransition` and whose `data.details` are the ones `decide` gives, `from` naming the status stored before the call and `trigger` naming ship; the depot's streams and events tables are read afterwards and equal their contents before the call. - The function log holds one attempt of the parent mutation, not rerun by the engine, and it failed with that rejection. diff --git a/design/specs/context/context-component.stale-version-rejected.sdp.md b/design/specs/context/context-component.stale-version-rejected.sdp.md index c7c2589..1b01d13 100644 --- a/design/specs/context/context-component.stale-version-rejected.sdp.md +++ b/design/specs/context/context-component.stale-version-rejected.sdp.md @@ -9,7 +9,7 @@ relations: --- # A command names a stale, explicitly reviewed version -Sc L1-10 · native tier · fixture composition. +Sc L1-10 · native backend tier · fixture composition. The fixture composition binds it. An ordinary client creates a document in the depot context, amends it and submits it, so its stream stands at version 3. It then sends the AmendDocument command through its public entry naming version 2, the version it reviewed, and the depot's amendDocuments operation plans the amend at that expected version. @@ -33,7 +33,7 @@ And the callers saw {saw: "a version conflict and no engine retry"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend; the caller is an ordinary client holding a fixture-issuer token and a grant, and admin access only grants and reads the depot's tables and the function log. +- Runs in the native backend tier; every test owns its disposable backend; the caller is an ordinary client holding a fixture-issuer token and a grant, and admin access only grants and reads the depot's tables and the function log. - The `ConvexError` carries `data.code` equal to `staleVersion` with `details.expected` of 2 and `details.current` of 3; the stream row still reads version 3, and the depot's streams and events tables equal their contents before the call. - In place of a spy on the decider, the same client sends the fixture's FailIfDecided command, whose `decide` throws a plain Error if it is reached, to the same document: naming version 2 it receives the same `staleVersion` data, and naming version 3 it fails with that Error, so the version check answers before `decide`, which is reachable at the current version; neither call changes the tables. - The function log holds one attempt of the amend's parent mutation, final and with no OCC conflict behind it, which is the no engine retry the caller saw. diff --git a/design/specs/context/journal.rebuild-from-baseline.sdp.md b/design/specs/context/journal.rebuild-from-baseline.sdp.md index 5c38dc2..988dc7b 100644 --- a/design/specs/context/journal.rebuild-from-baseline.sdp.md +++ b/design/specs/context/journal.rebuild-from-baseline.sdp.md @@ -9,7 +9,7 @@ relations: --- # Rebuild a stream that has a baseline event -Sc L2-7 · native tier. +Sc L2-7 · native backend tier. ## Intent @@ -28,6 +28,6 @@ And the number of events applied through evolve is {applied: 2} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend, the five earlier events are written by the old reducer's version of the context, and the new version registers a `BaselineMigration` on the stream type whose `migrate` maps the saved state to the new meaning and raises the registration's `stateSchemaVersion`. +- Runs in the native backend tier; every test owns its disposable backend, the five earlier events are written by the old reducer's version of the context, and the new version registers a `BaselineMigration` on the stream type whose `migrate` maps the saved state to the new meaning and raises the registration's `stateSchemaVersion`. - The test runs the same bound point under both orderings of the migration window. In the sweep-first run a parent test mutation calls `maintenance.writeBaseline` through `ctx.runMutation` with the migration's name before any command; the call returns `written` of 1 and `nextAfterStreamId` of `null`, and the two later commands append versions 7 and 8. In the command-first run a command reaches the stream before the driver: the adapter writes the baseline at 6 on load and the command's event at 7 in one sub-transaction, the second command appends 8, and the driver's batch, run afterwards, returns `written` of 0, `skipped` of 1 and `nextAfterStreamId` of `null`, so the sweep and the live command never race and no second baseline exists. - In both runs the `rebuild` query reports `fromBaseline` of 6, `eventsApplied` of 2 and `equal` true; the `history` query returns all eight events in version order, the first five with their original payloads and the sixth with `causedBy` of kind `migration`; and before the baseline is written in the command-first run the `rebuild` query answers `refused` with reason `awaitingBaseline`, never `equal` false. diff --git a/design/specs/decisions/d04-four-outcomes.sdp.md b/design/specs/decisions/d04-four-outcomes.sdp.md index 68ab04a..06997d3 100644 --- a/design/specs/decisions/d04-four-outcomes.sdp.md +++ b/design/specs/decisions/d04-four-outcomes.sdp.md @@ -31,4 +31,4 @@ A command ends in one of four outcomes, each with a fixed commit behavior. Appli - consequence: The `Outcome` union is defined once in the kernel and mapped to the wire once at the command boundary; no Spec defines a second one (D4) - consequence: A business failure is an event and a result, so it is journaled and visible to read models exactly like an applied command (D4) - consequence: A rejection is a thrown `ConvexError` at the public boundary and stores nothing, which D7 rules (D4, D7) -- consequence: The standing cost is one union type, one closed error-code list and a simulator tier that covers the outcome combinations (D4, Acceptance scenarios) +- consequence: The standing cost is one union type, one closed error-code list and a `convex-test` tier that covers the outcome combinations (D4, Acceptance scenarios) diff --git a/design/specs/effects/external-effects.provider-success-reply-lost.sdp.md b/design/specs/effects/external-effects.provider-success-reply-lost.sdp.md index 1d36039..fecd1c1 100644 --- a/design/specs/effects/external-effects.provider-success-reply-lost.sdp.md +++ b/design/specs/effects/external-effects.provider-success-reply-lost.sdp.md @@ -9,7 +9,7 @@ relations: --- # The provider succeeds but the reply is lost -Sc L3-5 · native tier. +Sc L3-5 · native backend tier. ## Intent @@ -28,7 +28,7 @@ And the provider key on the next attempt is {nextKey: "the same key"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The provider stub records the effect under the idempotency key and drops the reply, so the action's settle never runs; the test advances past the lease and runs `sweep`, which settles the attempt as ambiguous (D14, D13). - The test asserts the next claim carries the same `providerKey`, that the provider stub answers with the stored result, and that the obligation settles succeeded with `provider` evidence (D14, Sc L3-5). - The test repeats the scenario with policy `reconcile first` and asserts the reconcile attempt finds the effect and settles without a second call, and with policy `none` asserts the obligation is in needs attention with reason `uncertain` and no second call (D14). diff --git a/design/specs/effects/external-effects.stale-worker-after-cancellation.sdp.md b/design/specs/effects/external-effects.stale-worker-after-cancellation.sdp.md index d01d984..a0262e0 100644 --- a/design/specs/effects/external-effects.stale-worker-after-cancellation.sdp.md +++ b/design/specs/effects/external-effects.stale-worker-after-cancellation.sdp.md @@ -9,7 +9,7 @@ relations: --- # An old worker reports after a cancellation -Sc L3-6 · native tier · second of two cases: the report arrives after an operator cancelled the obligation. +Sc L3-6 · native backend tier · second of two cases: the report arrives after an operator cancelled the obligation. ## Intent @@ -28,7 +28,7 @@ And the obligation is {status: "cancelled"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test holds attempt 1's action past its lease so `expireLease` moves the obligation to needs attention under policy `none`, then cancels it as an operator, then releases the old action so its confirmed report arrives (D14, D13). - The test asserts the status is still cancelled, the report is in `lateEvidence`, `reconcileRequested` is true, and no new attempt or call was made (D14, Sc L3-6). - The test asserts `reconcile` as an operator with the provider's evidence is refused on a terminal obligation and that compensation must be a new operation, so the stub still shows one effect (D14, D13). diff --git a/design/specs/effects/external-effects.stale-worker-late-evidence.sdp.md b/design/specs/effects/external-effects.stale-worker-late-evidence.sdp.md index 37a1787..6982c87 100644 --- a/design/specs/effects/external-effects.stale-worker-late-evidence.sdp.md +++ b/design/specs/effects/external-effects.stale-worker-late-evidence.sdp.md @@ -9,7 +9,7 @@ relations: --- # An old worker reports after a new attempt -Sc L3-6 · native tier · first of two cases: the report arrives after a new attempt claimed the obligation. +Sc L3-6 · native backend tier · first of two cases: the report arrives after a new attempt claimed the obligation. ## Intent @@ -28,7 +28,7 @@ And the obligation is {status: "succeeded"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test holds attempt 1's action past its lease, lets `expireLease` claim a reconcile attempt, then releases the old action so its confirmed report reaches `settleAttempt` with the old attempt ID (D14). - The test asserts `settleAttempt` returned `stale`, the report is in `lateEvidence`, `reconcileRequested` is true, and the status and active attempt were not overwritten (D14, Sc L3-6). - The test asserts the reconcile attempt finds the provider's record under the same key and settles succeeded, with exactly one irreversible effect recorded by the stub (D14). diff --git a/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-mutation.sdp.md b/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-mutation.sdp.md index 8c45ad6..7be9dc3 100644 --- a/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-mutation.sdp.md +++ b/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-mutation.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 3: the cost of a component call from a mutation -Probe 3 · native tier · fixture composition. +Probe 3 · native backend tier · fixture composition. ## Intent @@ -24,7 +24,7 @@ And the local function log holds {recordsPerCall: 1} completion record for one p ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The execution time is the one the backend's function log reports for the parent call; the run's evidence holds the three medians and the difference per call. - The test checks that each path read the document the bound number of times before it accepts a sample. - The size is recorded and not asserted. It is a local backend's time on one machine, a first reading for OQ1 and not a hosted deployment's cost. diff --git a/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-query.sdp.md b/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-query.sdp.md index 49d172f..b589d10 100644 --- a/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-query.sdp.md +++ b/design/specs/facts/f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-query.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 3: the cost of a component call from a query -Probe 3 · native tier · fixture composition. +Probe 3 · native backend tier · fixture composition. ## Intent @@ -24,7 +24,7 @@ And the local function log holds {recordsPerCall: 1} completion record for one p ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The execution time is the one the backend's function log reports for the parent call, because the clock inside a query does not advance; the run's evidence holds the three medians and the difference per call. - The test checks that each path read the document the bound number of times before it accepts a sample. - A cached query also logs a completion record, with `cachedResult` true and a time near zero, so the test gives each call fresh arguments and counts a sample only when its record says the result was not cached. diff --git a/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-backend-restart-with-pending-mutation.sdp.md b/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-backend-restart-with-pending-mutation.sdp.md index 88258d8..ed434cb 100644 --- a/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-backend-restart-with-pending-mutation.sdp.md +++ b/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-backend-restart-with-pending-mutation.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 1: a backend restart with a mutation in flight -Probe 1 · native tier · fixture composition. +Probe 1 · native backend tier · fixture composition. ## Intent @@ -25,7 +25,7 @@ And at least {leastKilledBeforeCommit: 1} trial was killed before its commit and ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The client is `ConvexClient` from `convex/browser`, which sends mutations through the same `BaseConvexClient` as the React client; the React client itself is not run. - The kill is a SIGKILL, sent after a delay that the trials spread from none to a few tens of milliseconds. The test holds the backend's responses back from the client, reads the marker table with admin access after the restart and before the client reconnects, and so tells a kill before the commit from a kill after it. - A trial killed after its commit is the one that tests the guarantee: the client never saw the result, sends the mutation again when it reconnects, and the backend must not run it twice. diff --git a/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-client-closed-with-pending-mutation.sdp.md b/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-client-closed-with-pending-mutation.sdp.md index 862a8a4..381e65f 100644 --- a/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-client-closed-with-pending-mutation.sdp.md +++ b/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-client-closed-with-pending-mutation.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 1: a client closed with a pending mutation -Probe 1 · native tier · fixture composition. +Probe 1 · native backend tier · fixture composition. ## Intent @@ -24,7 +24,7 @@ And at least {leastCommittedUnconfirmed: 1} trial leaves its row although its cl ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The client is `ConvexClient` from `convex/browser`, which sends mutations through the same `BaseConvexClient` as the React client; the React client itself is not run. - The test holds the backend's responses back from the client, so every close happens with the mutation pending, and varies the time between the send and the close. - The test records how many trials left no row and how many left one, which is the measure of what a closed tab can lose. diff --git a/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-http-client-retry.sdp.md b/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-http-client-retry.sdp.md index 412e6d9..da027dd 100644 --- a/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-http-client-retry.sdp.md +++ b/design/specs/facts/f05-react-client-retries-until-confirmed.probe-1-http-client-retry.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 1: the HTTP client and a caller's retry -Probe 1 · native tier · fixture composition. +Probe 1 · native backend tier · fixture composition. ## Intent @@ -24,7 +24,7 @@ And the backend holds {markerRows: 2} marker rows ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The transport is the `fetch` the test passes to `ConvexHttpClient`: it performs the request, drops the response and throws, as a connection lost after the commit would. - The test also asserts that one call on an unfaulted transport sends one request and leaves one row. - On the first run of this example, on 2026-10-01 on release `precompiled-2026-09-28-5c7cb5b`, the bound values held. diff --git a/design/specs/facts/f13-transactions-have-limits.probe-4-component-call-shares-limits.sdp.md b/design/specs/facts/f13-transactions-have-limits.probe-4-component-call-shares-limits.sdp.md index 5091e95..49e9376 100644 --- a/design/specs/facts/f13-transactions-have-limits.probe-4-component-call-shares-limits.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.probe-4-component-call-shares-limits.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 4: a component call draws on the caller's transaction limits -Probe 4 · native tier · fixture composition. +Probe 4 · native backend tier · fixture composition. ## Intent @@ -25,7 +25,7 @@ And the same split of bytes written by a parent mutation and a component mutatio ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test also reads `ctx.meta.getTransactionMetrics()` in the parent before and after a component read that stays under the limit and asserts that `bytesRead.used` and `documentsRead.used` grew by what the component read. - The test also shows that each half of the write commits alone, so the failure is the sum. - The local backend refuses writes past a rate, with `TooManyWrites` and the text "limited to 4 MiB bytes written per 1 second", so the test paces the transactions that seed and write its documents. diff --git a/design/specs/facts/f13-transactions-have-limits.probe-4-nested-call-shares-limits.sdp.md b/design/specs/facts/f13-transactions-have-limits.probe-4-nested-call-shares-limits.sdp.md index d1d6094..383dbe4 100644 --- a/design/specs/facts/f13-transactions-have-limits.probe-4-nested-call-shares-limits.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.probe-4-nested-call-shares-limits.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 4: a nested call draws on the caller's transaction limits -Probe 4 · native tier · fixture composition. +Probe 4 · native backend tier · fixture composition. ## Intent @@ -25,7 +25,7 @@ And the same split of bytes written by a parent mutation and a nested mutation { ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test also reads `ctx.meta.getTransactionMetrics()` in the parent before and after a nested read that stays under the limit and asserts that `bytesRead.used` and `documentsRead.used` grew by what the child read. - The test also shows that each half of the write commits alone, so the failure is the sum. - The local backend refuses writes past a rate, with `TooManyWrites` and the text "limited to 4 MiB bytes written per 1 second", so the test paces the transactions that seed and write its documents. diff --git a/design/specs/facts/f13-transactions-have-limits.probe-4-nesting-depth.sdp.md b/design/specs/facts/f13-transactions-have-limits.probe-4-nesting-depth.sdp.md index f88305c..b3465fc 100644 --- a/design/specs/facts/f13-transactions-have-limits.probe-4-nesting-depth.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.probe-4-nesting-depth.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 4: how deep nested calls may go -Probe 4 · native tier · fixture composition. +Probe 4 · native backend tier · fixture composition. ## Intent @@ -24,6 +24,6 @@ And the first stack that does not commit is refused with an error whose text hol ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The value is observed, not expected: the expectation written before the first run was 8 functions, and the run on release `precompiled-2026-09-28-5c7cb5b` showed 9. - The bound is a limit the backend enforces, a default of this release and not a platform contract. Its error text names components, although the recursion crosses none. diff --git a/design/specs/facts/f14-convex-error-survives-nested-and-component-boundary.probe-2-component-boundary.sdp.md b/design/specs/facts/f14-convex-error-survives-nested-and-component-boundary.probe-2-component-boundary.sdp.md index ab2cb69..ed4b9df 100644 --- a/design/specs/facts/f14-convex-error-survives-nested-and-component-boundary.probe-2-component-boundary.sdp.md +++ b/design/specs/facts/f14-convex-error-survives-nested-and-component-boundary.probe-2-component-boundary.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 2: ConvexError data through a component boundary -Probe 2 · native tier · fixture composition. +Probe 2 · native backend tier · fixture composition. ## Intent @@ -26,7 +26,7 @@ And after the catching parent committed, the component mutation's table holds {c ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The thrown data holds a string, a number, a boolean, a null, an array, a nested object and a 64-bit integer, and the test compares decoded values, not messages. - The test shows that the thrower had written its row before it threw, so a thrower that never wrote does not pass. - The test also throws an ordinary `Error` with an added property through the same path, checks that the error it sees is the thrower's, and records whether the property arrives. diff --git a/design/specs/facts/f14-convex-error-survives-nested-and-component-boundary.probe-2-nested-mutation.sdp.md b/design/specs/facts/f14-convex-error-survives-nested-and-component-boundary.probe-2-nested-mutation.sdp.md index 6de03f3..621e557 100644 --- a/design/specs/facts/f14-convex-error-survives-nested-and-component-boundary.probe-2-nested-mutation.sdp.md +++ b/design/specs/facts/f14-convex-error-survives-nested-and-component-boundary.probe-2-nested-mutation.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 2: ConvexError data through a nested mutation -Probe 2 · native tier · fixture composition. +Probe 2 · native backend tier · fixture composition. ## Intent @@ -26,7 +26,7 @@ And after the catching parent committed, the throwing mutation's table holds {ch ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The thrown data holds a string, a number, a boolean, a null, an array, a nested object and a 64-bit integer, and the test compares decoded values, not messages. - The test shows that the thrower had written its row before it threw, so a thrower that never wrote does not pass. - The test also throws an ordinary `Error` with an added property through the same path, checks that the error it sees is the thrower's, and records whether the property arrives. diff --git a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-built-in-paginate-throws-in-component.sdp.md b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-built-in-paginate-throws-in-component.sdp.md index b2cd902..79ba0a5 100644 --- a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-built-in-paginate-throws-in-component.sdp.md +++ b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-built-in-paginate-throws-in-component.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 5: the built-in paginate throws inside a component -Probe 5 · native tier · fixture composition. +Probe 5 · native backend tier · fixture composition. ## Intent @@ -23,5 +23,5 @@ Then the read fails with an error that says paginate is only supported in the ap ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test reads the component query through the parent query `list:builtinPage` and asserts that the error's message contains `paginate() is only supported in the app`, and that the parent query `list:page`, which relays the component's query built with `paginator` on the same table, returns its page. diff --git a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split.sdp.md b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split.sdp.md index 74f3696..e41f505 100644 --- a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split.sdp.md +++ b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-not-split.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 5: a full first page with a row cap of twice its size is not split -Probe 5 · native tier · fixture composition. +Probe 5 · native backend tier · fixture composition. ## Intent @@ -23,6 +23,6 @@ Then the page holds {rowsHeld: 10} rows and carries pageStatus {pageStatus: "non ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - A `pageStatus` of none means the relayed page has no `pageStatus` or has it `null`; the test asserts `isDone` false and a `continueCursor` that starts the next page at the eleventh row. - The sibling example with the row cap equal to the page size shows the reading this one rules out. diff --git a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap.sdp.md b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap.sdp.md index 3089c1a..31a6cbe 100644 --- a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap.sdp.md +++ b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-full-first-page-split-at-equal-cap.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 5: a full first page with a row cap equal to its size comes back split -Probe 5 · native tier · fixture composition. +Probe 5 · native backend tier · fixture composition. ## Intent @@ -23,5 +23,5 @@ Then the page holds {rowsHeld: 10} rows and carries pageStatus {pageStatus: "Spl ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that the relayed page carries a `splitCursor`, so the status is the helper's and not an artifact of the relay. diff --git a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-hook-splits-across-boundary.sdp.md b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-hook-splits-across-boundary.sdp.md index 58e0b76..bd87c14 100644 --- a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-hook-splits-across-boundary.sdp.md +++ b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-hook-splits-across-boundary.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 5: the client hook splits a page across the boundary and loses rows -Probe 5 · native tier · fixture composition. +Probe 5 · native backend tier · fixture composition. ## Intent @@ -25,7 +25,7 @@ And {rowsMissing: 5} rows of the list are not shown, the ones between the capped ## Verification — executable -- Runs in the native tier on the fixture composition, with a DOM environment for the hook; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition, with a DOM environment for the hook; every test owns its disposable backend. - The test shows through a second subscription that the capped page came back `SplitRequired`, so a pass cannot come from a page that never needed the split. - The hook is the `usePaginatedQuery` of `convex-helpers/react`, whose pages carry an end cursor. Convex's own `usePaginatedQuery` is not the subject. - The values are observed, not expected: the expectation written before the first run was 45 rows shown and none missing. The run on release `precompiled-2026-09-28-5c7cb5b` with `convex-helpers` 0.1.124 showed 40. diff --git a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-pages-stay-contiguous.sdp.md b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-pages-stay-contiguous.sdp.md index ab9d811..75b157d 100644 --- a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-pages-stay-contiguous.sdp.md +++ b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-pages-stay-contiguous.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 5: pages of a component list stay contiguous under live writes -Probe 5 · native tier · fixture composition. +Probe 5 · native backend tier · fixture composition. ## Intent @@ -25,6 +25,6 @@ And the first page now holds {firstPageRows: 15} rows ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test also deletes a row inside the second page's range and asserts the same contiguity. - On the first run of this example, on 2026-10-01 on release `precompiled-2026-09-28-5c7cb5b`, the bound values held. diff --git a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap.sdp.md b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap.sdp.md index 7adac13..22657e8 100644 --- a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap.sdp.md +++ b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-parent-page-outgrows-row-cap.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 5: a pinned page of a parent table outgrows its row cap and comes back whole -Probe 5 · native tier · fixture composition. +Probe 5 · native backend tier · fixture composition. ## Intent @@ -24,6 +24,6 @@ And the page from the start to the split cursor and the page from the split curs ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The table is a table of the fixture composition's parent, read by a parent query with no component call, so the subject is the built-in call that a read-model list makes. - The test subscribes to the first page with the `endCursor` its first read returned, which is how a client pins a page. diff --git a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-split-required-relayed.sdp.md b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-split-required-relayed.sdp.md index 11caa05..d0738fb 100644 --- a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-split-required-relayed.sdp.md +++ b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-split-required-relayed.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 5: a page that outgrows its read cap comes back through the parent as a split -Probe 5 · native tier · fixture composition. +Probe 5 · native backend tier · fixture composition. ## Intent @@ -24,7 +24,7 @@ And the page from the start to the split cursor and the page from the split curs ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - This example reads the relayed pages directly. Whether the client hook acts on the split is the next example's question. - The second half ends at the end cursor the page was subscribed with. The capped page's own `continueCursor` sits at the last row it read, and a second half that ends there holds 20 rows and leaves the rest of the range out. - On the first run of this example, on 2026-10-01 on release `precompiled-2026-09-28-5c7cb5b`, the bound values held. diff --git a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-subscription-updates.sdp.md b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-subscription-updates.sdp.md index c245bfa..45cf18e 100644 --- a/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-subscription-updates.sdp.md +++ b/design/specs/facts/f15-parent-query-over-component-query-stays-reactive.probe-5-subscription-updates.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 5: a subscription to a parent query follows the component's data -Probe 5 · native tier · fixture composition. +Probe 5 · native backend tier · fixture composition. ## Intent @@ -23,6 +23,6 @@ Then the subscription delivers {updates: 1} changed value without the client ask ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The client is a `ConvexClient` over a WebSocket, and the test fails if no update arrives within a stated wait. - On the first run of this example, on 2026-10-01 on release `precompiled-2026-09-28-5c7cb5b`, the bound values held. diff --git a/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md b/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md index 0919ff8..f8e0a13 100644 --- a/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md +++ b/design/specs/facts/f18-commit-timestamps.probe-8-parent-and-components.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 8: commit timestamps in the parent and two components -Probe 8 · native tier · fixture composition. +Probe 8 · native backend tier · fixture composition. ## Intent @@ -24,7 +24,7 @@ And returning a placeholder {returned: "resolves to a bigint"} and scheduling it ## Verification — executable -- Runs in the native tier on the fixture composition, where `annex` is mounted twice so that two components write; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition, where `annex` is mounted twice so that two components write; every test owns its disposable backend. - Inside the writing mutation the test asserts that a read of the row yields the unresolved placeholder, in the parent and in each component, and in each component that converting it to a number throws. - The expectation written before the first run was that returning the placeholder is refused; the pinned release returned the committed bigint, and the example is bound to that observed value. The refusal to schedule it held, with `Field name $commitTs starts with '$', which is reserved.` - The query uses the internal runtime `getSnapshotTs()` through an explicit cast. A compiled type assertion checks that the public `QueryMeta` declaration has no such method. A local backend cannot say whether a hosted deployment behaves the same, and no public upper bound is shown; both stay open on F18. diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md index 52ddb4e..8505fd4 100644 --- a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-11-component-calls.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 11: component calls per mutation -Probe 11 · native tier · fixture composition. +Probe 11 · native backend tier · fixture composition. ## Intent @@ -24,7 +24,7 @@ And each empty component call reads {reads: 0} and writes {writes: 0} documents ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. Times and call counts are recorded, not asserted, and say nothing about a hosted deployment. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. Times and call counts are recorded, not asserted, and say nothing about a hosted deployment. - The expectation written before the first run, a refusal distinct from the one-second execution limit, held; the example binds the full error text that run showed. A run that never reaches a refusal fails. - Successful call counts, the first refused requested count, wall time and top-level completion time are recorded in the run's evidence, marked as taken under load unless `LIBAR_TIMINGS_UNDER_LOAD=0` says the machine was otherwise idle. The failed request cannot return its exact completed-call count. - A twenty-call control also inserts and reads one document per component call, asserts one read and one write per call, and records transaction metrics and completion usage. The example verifies F19 only; it does not compare against a plain helper for F4. diff --git a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md index 361944f..9370246 100644 --- a/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md +++ b/design/specs/facts/f19-nested-calls-share-time-budgets.probe-9-nested-time.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 9: nested calls reach a time boundary -Probe 9 · native tier · fixture composition. +Probe 9 · native backend tier · fixture composition. ## Intent @@ -24,6 +24,6 @@ And a loop of computation in one mutation fails with {cpu: "Function execution t ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. Times and call counts are recorded, not asserted, and say nothing about a hosted deployment. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. Times and call counts are recorded, not asserted, and say nothing about a hosted deployment. - The expectation written before the first run, a refusal distinct from the one-second execution limit, held; the example binds the full error text that run showed. A run that never reaches a refusal fails. - Successful call counts, the first refused requested count, wall time and top-level completion time are recorded in the run's evidence, marked as taken under load unless `LIBAR_TIMINGS_UNDER_LOAD=0` says the machine was otherwise idle. The failed request cannot return its exact completed-call count. diff --git a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md index 5b095c1..7098b72 100644 --- a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md +++ b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-oversized-row.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 10: a page keeps the row that crosses its byte bound -Probe 10 · native tier · fixture composition. +Probe 10 · native backend tier · fixture composition. ## Intent @@ -24,6 +24,6 @@ And the bytes read are above every bound below the row's size {overshoot: true} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. The stored size of the row and the bytes each page read, from `ctx.meta.getTransactionMetrics()`, are recorded in the run's evidence. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. The stored size of the row and the bytes each page read, from `ctx.meta.getTransactionMetrics()`, are recorded in the run's evidence. - The bound values are the expectation written before the first run, which held. The bounds are zero, one, one byte below the stored size and the stored size; a page that kept no row would not have reached the boundary, and the test fails it. - The test also supplies a negative byte bound. The helper still reads and keeps one row, so the overshoot guarantee is stated only for finite non-negative bounds. diff --git a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md index ec0c34b..30a309b 100644 --- a/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md +++ b/design/specs/facts/f20-pagination-checks-bytes-after-reading.probe-10-stream-budget.sdp.md @@ -9,7 +9,7 @@ relations: --- # Probe 10: a page of full stream rows reads past the library's byte bound -Probe 10 · native tier · fixture composition. +Probe 10 · native backend tier · fixture composition. ## Intent @@ -24,7 +24,7 @@ And the bytes the page read are above the library's byte bound {exceeds: true} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test uses `boundedPage`, `limitListPage` and `limitListBytes` from `src/context/queries.ts`, with an end cursor that removes the item count as a stopping condition. - The bound values are the expectation written before the first run. Stored sizes and measured bytes are recorded, not bound as constants. - The table has the same application-field budget as a default single-mapping stream. Its stored rows also carry `_id` and `_creationTime`, which the helper includes in the byte count. The row cap of twice the item cap is reached on the same row as the byte bound, so the example shows what the page read and not which cap stopped it. diff --git a/design/specs/kernel/domain-kernel.evaluate-twice.sdp.md b/design/specs/kernel/domain-kernel.evaluate-twice.sdp.md index 4544a6b..8a0e6f0 100644 --- a/design/specs/kernel/domain-kernel.evaluate-twice.sdp.md +++ b/design/specs/kernel/domain-kernel.evaluate-twice.sdp.md @@ -9,7 +9,7 @@ relations: --- # Evaluate a decision twice with identical inputs -Sc L0-1 · domain tier. +Sc L0-1 · pure test tier. ## Intent @@ -27,5 +27,5 @@ And the number of I/O calls observed is {io: 0} ## Verification — executable -- Runs in the domain tier; no backend is involved and the test imports the kernel package only. +- Runs in the pure test tier; no backend is involved and the test imports the kernel package only. - The inputs are deep-frozen before the first evaluation and compared by deep equality after the second; the outputs are compared by deep equality; a spy on the database, network, scheduler, environment and auth surfaces records zero calls. diff --git a/design/specs/kernel/domain-kernel.incremental-then-rebuild.sdp.md b/design/specs/kernel/domain-kernel.incremental-then-rebuild.sdp.md index 12cfd5a..b02fe9a 100644 --- a/design/specs/kernel/domain-kernel.incremental-then-rebuild.sdp.md +++ b/design/specs/kernel/domain-kernel.incremental-then-rebuild.sdp.md @@ -9,7 +9,7 @@ relations: --- # Run commands incrementally, then rebuild the stream from its events -Sc L0-2 · domain tier. +Sc L0-2 · pure test tier. ## Intent @@ -27,5 +27,5 @@ And the number of I/O calls observed is {io: 0} ## Verification — executable -- Runs in the domain tier; no backend is involved and the test imports the kernel package only. +- Runs in the pure test tier; no backend is involved and the test imports the kernel package only. - Each command is decided against the state folded from the previous commands' events, never against a replay of history; the rebuild folds `evolve` from `initial()` over the collected events, and the version is the count of events. diff --git a/design/specs/kernel/domain-kernel.sdp.md b/design/specs/kernel/domain-kernel.sdp.md index dfe5c02..b6b7a59 100644 --- a/design/specs/kernel/domain-kernel.sdp.md +++ b/design/specs/kernel/domain-kernel.sdp.md @@ -18,7 +18,7 @@ relations: Layer 0 · Detail: full · Traces: D3, D4, D5, Law 3, Law 10, Sc L0-1, Sc L0-2, E-1, E-20. -The kernel is the pure TypeScript module of a bounded context: its commands, events, `decide`, `evolve`, initial state, invariants and small state machines. It owns the domain decision and nothing else. It reads no database, network, scheduler, environment or ambient auth; time and outside facts enter as inputs; the next state is `fold(evolve, state, newEvents)` and nothing else. The kernel is usable before any Convex code exists, which is why its two scenarios run in the domain tier with no backend at all. +The kernel is the pure TypeScript module of a bounded context: its commands, events, `decide`, `evolve`, initial state, invariants and small state machines. It owns the domain decision and nothing else. It reads no database, network, scheduler, environment or ambient auth; time and outside facts enter as inputs; the next state is `fold(evolve, state, newEvents)` and nothing else. The kernel is usable before any Convex code exists, which is why its two scenarios run in the pure test tier with no backend at all. Its boundary is the decider contract. Above it sits the context component of Layer 1, whose persistence adapter loads current state, calls `decide`, folds the new events, appends them and saves the result inside one component sub-transaction. Below it sits nothing. The kernel imports no Convex module; the context boundary supplies validators for command input and event payloads, and the adapter supplies the decision context. @@ -27,7 +27,7 @@ Its boundary is the decider contract. Above it sits the context component of Lay - actor: The domain author who writes a context's decider, and the persistence adapter that calls it (D3) - problem: Today's platform has two authorities for state, a `stateUpdate` the handler saves and an `evolve` beside it, and nothing keeps them in step; a decider that reads the clock or a table cannot be evaluated twice with the same answer, and a rebuild from the creation event has no defined start (D3, Sc L0-1, Sc L0-2) - outcome: A decider is a pure function of state, command and decision context that returns events and a result or a rejection; the next state is the fold of `evolve` over the new events; rebuilding a stream from its events gives the saved business state and the same stream version, and that equality is a test (D3, Law 3, Sc L0-2) -- value: State and history agree by construction, domain tests need no backend, and a decision can be reasoned about, replayed and refuted without infrastructure (D3) +- value: State and history agree by construction, pure tests need no backend, and a decision can be reasoned about, replayed and refuted without infrastructure (D3) - risk: Every change to `evolve` must still turn old events into the state saved today; the tool that pays for a change of meaning is the baseline event, whose cost D5 carries (D3, D5) - risk: The contract fixes a shape that every context must fit; a domain that needs a decision across several streams at once does not fit it and waits for its trigger (D3, D18) - assumption: The four outcomes and the reserved rejection codes are the ones the outcome model defines; the kernel adds no fifth (D4) @@ -71,8 +71,8 @@ The kernel is one TypeScript package per context, or one module inside the conte - convexTypeImport: the kernel package lists `convex` under `devDependencies` only and compiles with `verbatimModuleSyntax`, so `import type { Value } from "convex/values"` is the only form the compiler accepts for the type and the emitted module carries no `convex` import; a value import of `convex/values` or `convex/server` compiles and is refused by the ESLint rule `@typescript-eslint/no-restricted-imports` over the kernel's sources, which CI runs (E-20, E-21, D3) - moduleShape: one module per context exporting its deciders and types, plus the shared kernel library that exports the contract types and `fold`, `rebuild`, `transition` and `checkInvariants` (D3, E-20) - actorView: the decision context's actor is the kernel's structural `DecisionActor`, which the command family's `Actor` satisfies; the kernel therefore depends on no Layer 1 Spec and stays usable before Layer 1 exists (E-20, Thesis) -- purityCheck: a domain test evaluates `decide` twice on frozen inputs and asserts deep equality of the outputs and of the inputs before and after, with an I/O spy that observes zero calls (Sc L0-1) -- rebuildCheck: a domain test runs a command sequence incrementally through `decide` and `fold`, then folds `evolve` from `initial()` over the collected events and asserts deep equality of state and equality of the event count with the stream version (Sc L0-2) +- purityCheck: a pure test evaluates `decide` twice on frozen inputs and asserts deep equality of the outputs and of the inputs before and after, with an I/O spy that observes zero calls (Sc L0-1) +- rebuildCheck: a pure test runs a command sequence incrementally through `decide` and `fold`, then folds `evolve` from `initial()` over the collected events and asserts deep equality of state and equality of the event count with the stream version (Sc L0-2) - eventShape: a domain event carries `eventType`, `eventSchemaVersion`, `payload` and an optional `occurredAt`; the envelope fields of D2 are added by the adapter, never by the decider (D2, E-20) - outcomeSource: the `Outcome` union and the reserved rejection codes are defined in `spec:kernel.outcome-model` and reused here (D4, E-21) - multiStreamDecisions: a decision that must hold against several streams at once is out of the kernel's scope until the explicit-scope trigger of D18 fires; until then an operation decides per stream and the parent or the context's plan sequences the streams (D18, D10) diff --git a/design/specs/kernel/outcome-model.sdp.md b/design/specs/kernel/outcome-model.sdp.md index 36f20a4..eaea3ac 100644 --- a/design/specs/kernel/outcome-model.sdp.md +++ b/design/specs/kernel/outcome-model.sdp.md @@ -72,4 +72,4 @@ The union below is the kernel's. Applied and business failure are the value a co ## Verification — reviewed - A reviewer confirms that no other Spec in the corpus defines a second outcome union and that `spec:command.outcome-boundary` maps from this one. -- A reviewer confirms that the simulator tier covers every outcome kind and the three reserved codes. +- A reviewer confirms that the `convex-test` tier covers every outcome kind and the three reserved codes. diff --git a/design/specs/obligations/obligation-module.backlog-left-alone.sdp.md b/design/specs/obligations/obligation-module.backlog-left-alone.sdp.md index cb5744c..90924d3 100644 --- a/design/specs/obligations/obligation-module.backlog-left-alone.sdp.md +++ b/design/specs/obligations/obligation-module.backlog-left-alone.sdp.md @@ -9,7 +9,7 @@ relations: --- # Legitimate backlog builds up -Sc L3-3 · native tier. +Sc L3-3 · native backend tier. ## Intent @@ -28,7 +28,7 @@ And the queued dispatch is {queued: "left alone"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test creates enough scheduled work ahead of the dispatch that its `_scheduled_functions` row stays Pending past the grace, then runs `sweep` (F16). - The test asserts that the obligation's `activeAttemptId`, `dispatchId` and `rearmCount` are unchanged after the sweep, and that the sweep's counts show it as left alone (D13, Sc L3-3). - The test asserts that once the backlog drains the original dispatch runs and the obligation succeeds with one effect (D13). diff --git a/design/specs/obligations/obligation-module.dispatch-killed-rearmed.sdp.md b/design/specs/obligations/obligation-module.dispatch-killed-rearmed.sdp.md index 1279876..b2ff442 100644 --- a/design/specs/obligations/obligation-module.dispatch-killed-rearmed.sdp.md +++ b/design/specs/obligations/obligation-module.dispatch-killed-rearmed.sdp.md @@ -9,7 +9,7 @@ relations: --- # Kill a dispatch before work -Sc L3-2 · native tier · first of two cases: the dispatch is cancelled before the wrapper runs. +Sc L3-2 · native backend tier · first of two cases: the dispatch is cancelled before the wrapper runs. ## Intent @@ -27,7 +27,7 @@ And a bounded rearm {rearm: "follows"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test cancels the first dispatch with `ctx.scheduler.cancel` before it starts, so its `_scheduled_functions` row is Canceled, then advances past the grace and runs `sweep` (F16, S6). - The test asserts, before the rearmed dispatch executes, that the obligation is pending with `rearmCount` 1, a new `activeAttemptId`, the same `attemptNumber`, and a `dispatchId` whose row is Pending (D13). - The test then lets the rearmed dispatch run and asserts the obligation succeeded with one effect, and that a sixth kill in a row moves it to needs attention with reason `recoveryFailing` (D13, Sc L3-4). diff --git a/design/specs/obligations/obligation-module.duplicate-worker-lost-callback.sdp.md b/design/specs/obligations/obligation-module.duplicate-worker-lost-callback.sdp.md index 1cb215b..5d990c6 100644 --- a/design/specs/obligations/obligation-module.duplicate-worker-lost-callback.sdp.md +++ b/design/specs/obligations/obligation-module.duplicate-worker-lost-callback.sdp.md @@ -9,7 +9,7 @@ relations: --- # Duplicate a local worker and lose any completion callback -Sc L3-1 · native tier. +Sc L3-1 · native backend tier. ## Intent @@ -28,7 +28,7 @@ And the truth about completion comes from {truth: "the obligation record"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test invokes `runAttempt` twice with the same obligation ID and attempt ID; the second invocation returns without a write because the fence on `activeAttemptId` refuses it (D13). - The test drops any completion notification and asserts that `status`, `completionEvidence` and the body's events are consistent by reading the obligation and the journal, never a callback (Law 7). - The test asserts exactly one event set was appended by the body, by counting events under the obligation's operation ID (D13, D2). diff --git a/design/specs/obligations/obligation-module.exhausted-needs-attention.sdp.md b/design/specs/obligations/obligation-module.exhausted-needs-attention.sdp.md index f567e54..0559066 100644 --- a/design/specs/obligations/obligation-module.exhausted-needs-attention.sdp.md +++ b/design/specs/obligations/obligation-module.exhausted-needs-attention.sdp.md @@ -9,7 +9,7 @@ relations: --- # Exhaust retries, including failures of recovery itself -Sc L3-4 · native tier. +Sc L3-4 · native backend tier. ## Intent @@ -29,7 +29,7 @@ And the operator exit {exit: "works"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test drives the body to throw a transient error on every attempt until `attemptCount` reaches `maxAttempts`, and asserts the obligation is in needs attention with reason `exhausted`, no event committed, and `lastError` of class `retryable` (D13, F3). - The test separately drives `recoverOne` to throw on every rearm and asserts that the rearm count still climbs and the obligation reaches needs attention with reason `recoveryFailing` (D13, Sc L3-4). - The test calls `retry`, `cancel` and `abandon` as an operator and asserts each allowed transition writes a repair and each refused one writes nothing (D13, Law 8). diff --git a/design/specs/obligations/obligation-module.wrapper-failed-rearmed.sdp.md b/design/specs/obligations/obligation-module.wrapper-failed-rearmed.sdp.md index 97a29e2..25aa171 100644 --- a/design/specs/obligations/obligation-module.wrapper-failed-rearmed.sdp.md +++ b/design/specs/obligations/obligation-module.wrapper-failed-rearmed.sdp.md @@ -9,7 +9,7 @@ relations: --- # Fail the scheduled wrapper -Sc L3-2 · native tier · second of two cases: the wrapper itself throws outside the body's catch. +Sc L3-2 · native backend tier · second of two cases: the wrapper itself throws outside the body's catch. ## Intent @@ -27,7 +27,7 @@ And a bounded rearm {rearm: "follows"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test injects a developer error into the wrapper after the body returned and before the settle patch, so the whole scheduled mutation rolls back and its `_scheduled_functions` row is Failed (F9, F3). - The test asserts that no event of the body and no patch of the obligation survived, that the obligation is still pending on the old attempt ID, and that `sweep` rearms it once the grace has passed (D13, Sc L3-2). - The test asserts the rearm is bounded: after the rearm bound is reached the obligation is in needs attention with reason `recoveryFailing` and `retry` works from there (D13, Law 8). diff --git a/design/specs/obligations/retention-and-restore.restore-behind-provider.sdp.md b/design/specs/obligations/retention-and-restore.restore-behind-provider.sdp.md index a5225b5..5b44d06 100644 --- a/design/specs/obligations/retention-and-restore.restore-behind-provider.sdp.md +++ b/design/specs/obligations/retention-and-restore.restore-behind-provider.sdp.md @@ -9,7 +9,7 @@ relations: --- # Restore while provider state has moved past the backup -Sc L3-7 · end to end tier. +Sc L3-7 · native backend tier · production composition. ## Intent @@ -28,7 +28,7 @@ And irreversible provider effects number {effects: 1} ## Verification — executable -- Runs in the end to end tier; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - The test takes a backup with the obligation running under policy `reconcile first`, lets the provider stub confirm the effect after the backup, restores with `OBLIGATIONS_DISPATCH` off, and asserts no provider call is made and no call-mode claim proceeds until the switch is on, while the reconcile attempt that `rebuildSchedules` claims runs with the switch off (D19, F12). - The test asserts the obligation's `_id`, `effectKey` and `providerKey` after restore equal those before, and that the reconcile pass settles it succeeded from the provider's evidence without a second call (D19, D14). - The test runs `sweep` for several ticks, more than the rearm bound, with the switch still off and a second, pending external obligation in the backup, and asserts that this obligation's `rearmCount`, `attemptNumber` and status are unchanged after the ticks, that its dispatch row is a pending `claimAttempt` with its own attempt ID, and that it settles succeeded within a minute of the switch turning on (D19, E-56, Sc L3-7). diff --git a/design/specs/obligations/retention-and-restore.retention-keeps-unresolved.sdp.md b/design/specs/obligations/retention-and-restore.retention-keeps-unresolved.sdp.md index f37ed6d..002a4c3 100644 --- a/design/specs/obligations/retention-and-restore.retention-keeps-unresolved.sdp.md +++ b/design/specs/obligations/retention-and-restore.retention-keeps-unresolved.sdp.md @@ -9,7 +9,7 @@ relations: --- # Retention runs while retries or redelivery are still possible -Sc L3-8 · native tier. +Sc L3-8 · native backend tier. ## Intent @@ -28,7 +28,7 @@ And unresolved work {unresolved: "is untouched"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test seeds a succeeded obligation settled 3 days ago, a pending one, a running one and one in needs attention, sets the registry's longest validity to 30 days, and runs `retain` to completion (D13, D19). - The test asserts the succeeded row and its `completionEvidence` still exist, that the three unresolved rows are byte-identical to their seeds, and that a retried attempt with the same effect key still finds the existing row (D13, Sc L3-8). - The test advances 31 days, runs `retain` again, and asserts only the succeeded row is deleted (D13, E-55). diff --git a/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md b/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md index e16b2b2..d2cefc6 100644 --- a/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md +++ b/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md @@ -9,7 +9,7 @@ relations: --- # Break mandatory audit -Sc ALL-1 · native tier · the second of the two enumerated cases; applies to every installed layer. +Sc ALL-1 · native backend tier · the second of the two enumerated cases; applies to every installed layer. ## Intent @@ -25,6 +25,6 @@ And the failure is {surfaced: "returned to the caller as a technical failure"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test points the fixture app's audit writer at a table whose validator refuses the record and runs `PlaceOrder`. - The test asserts that the caller sees a technical failure, not a rejection code, and that no order, event, receipt, summary row or audit record exists for the operation. diff --git a/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md b/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md index b666611..70da684 100644 --- a/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md +++ b/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md @@ -9,7 +9,7 @@ relations: --- # Break metrics and logging -Sc ALL-1 · native tier · the first of the two enumerated cases; applies to every installed layer. +Sc ALL-1 · native backend tier · the first of the two enumerated cases; applies to every installed layer. ## Intent @@ -25,6 +25,6 @@ And the failure is {surfaced: "reported as a diagnostic gap without touching the ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test replaces the fixture app's diagnostic sink with one that throws on every call and runs `PlaceOrder`. - The test asserts that the command returned applied, that the order, its events, its receipt, its summary row and its audit record exist, and that the fixture's swallowed-error counter is one. diff --git a/design/specs/platform/acceptance-contract.sdp.md b/design/specs/platform/acceptance-contract.sdp.md index 5d40229..5d8182a 100644 --- a/design/specs/platform/acceptance-contract.sdp.md +++ b/design/specs/platform/acceptance-contract.sdp.md @@ -10,18 +10,18 @@ relations: Feature · Detail: full transcription · Traces: Acceptance scenarios (table, tiers, evidence), First experiment, S13, Sc L2-4, E-14, E-15, E-17. -The doc's acceptance table is the contract. It lists forty-three scenario rows by layer with a pass condition each, and states how each row is tested (the tier), what a test run must record (the evidence) and how readiness is stated. This Spec carries those rules. The rows themselves become example Specs under the component that first provides the capability, one example per row and one sibling example per enumerated case, so that each pass condition is a bound point in that component's example space; the plan's section 4 gives the assignment and the example IDs. How a native test obtains an identity, what the fixture app contains and what a test controls are the rules of `spec:platform.native-harness`, which refines this Spec. +The doc's acceptance table is the contract. It lists forty-three scenario rows by layer with a pass condition each, and states how each row is tested (the tier), what a test run must record (the evidence) and how readiness is stated. This Spec carries those rules. The rows themselves become example Specs under the component that first provides the capability, one example per row and one sibling example per enumerated case, so that each pass condition is a bound point in that component's example space; the plan's section 4 gives the assignment and the example IDs. How a native backend test obtains an identity, what the fixture app contains and what a test controls are the rules of `spec:platform.native-harness`, which refines this Spec. ## Intent - outcome: Every acceptance scenario is tested as behavior on the tier that can prove it, with recorded evidence, and readiness is stated in plain terms that a run can back (Acceptance scenarios) - value: The scenarios are the spine of the decision method: a mechanism is justified only by a scenario the do-nothing option fails, so the contract is also the refutation surface for every mechanism (Decision method rule 3) -- risk: A native tier costs a disposable backend per test and a production-configuration run; skipping it leaves component, nested-mutation, scheduling and contention behavior proven only in a simulator, which is not native proof (Acceptance scenarios) -- assumption: Native tests run against a local Convex backend as S13 describes (S13) +- risk: The native backend tier costs a disposable backend per test and a production-configuration run; skipping it leaves component, nested-mutation, scheduling and contention behavior proven only in `convex-test`, which is not native backend proof (Acceptance scenarios) +- assumption: Native backend tests run against a local Convex backend as S13 describes (S13) ### Open questions -- [non-blocking] Extension E-14: the doc's four tiers, domain, simulator, native and end to end, name no compile-time check, while Sc L2-4's pass condition is a build, type or registration check that fails before any traffic; the option taken here is a fifth tier, the build tier, a compile, with no backend, of a typed caller that states the change, which asserts that `tsc` refuses it, run with every type check and recorded like any run; the alternative, folding it into the native tier as a compile step that precedes the native run, was not taken because a native run needs a backend and the check needs none; the owner confirms (E-14, Sc L2-4, Acceptance scenarios) +- [non-blocking] Extension E-14: the doc's four tiers, domain, simulator, native and end to end, name no compile-time check, while Sc L2-4's pass condition is a build, type or registration check that fails before any traffic; the option taken here is a fifth tier, the compiled tier, a compile, with no backend, of a typed caller that states the change, which asserts that `tsc` refuses it, run with every type check and recorded like any run; the alternative, folding it into the native backend tier as a compile step that precedes the native run, was not taken because a native run needs a backend and the check needs none; the owner confirms (E-14, Sc L2-4, Acceptance scenarios) - [non-blocking] Extension E-17: the doc says that a link between source and test is not a passing run and that the first experiment passes when every Layer 0, 1 and 2 scenario passes, and it does not say how a pass of the whole is established from runs; the option taken here is the acceptance check: the required scenarios are generated from the graph, one per example of a required row with its tier, composition and verifiers, and read against one run's record, so that a scenario with no verifier, with no result or with a result other than passed keeps the whole from passing; the alternative, a list of scenarios and verifiers kept by hand beside the tests, was not taken because the graph already holds each example's row, tier, composition and verifiers; the owner confirms (E-17, Acceptance scenarios, First experiment) ## Rule @@ -29,21 +29,21 @@ The doc's acceptance table is the contract. It lists forty-three scenario rows b - Scenarios test behavior, not package names or the number of layers (Acceptance scenarios) - A scenario's layer is the first layer that makes the capability available (Acceptance scenarios) - Every scenario carries a pass condition, and the run passes only when that condition holds on the scenario's tier (Acceptance scenarios) -- Domain tests are pure (Acceptance scenarios) -- Simulator tests cover outcome combinations, serialization and classification (Acceptance scenarios) -- Native tests prove component and nested-mutation behavior, scheduling, contention and deployment (Acceptance scenarios, S13) -- A small end-to-end path runs the production composition (Acceptance scenarios) +- The pure test tier holds the doc's domain tests, which are pure (Acceptance scenarios) +- `convex-test` tests cover outcome combinations, serialization and classification (Acceptance scenarios) +- Native backend tests prove component and nested-mutation behavior, scheduling, contention and deployment (Acceptance scenarios, S13) +- A small end-to-end path runs the production composition in the native backend tier (Acceptance scenarios) - The kernel's fixture app is separate from the example app (Acceptance scenarios) - Test-only functions never ship in production (Acceptance scenarios) - Every test owns its disposable backend (Acceptance scenarios) - A run records the commit, installed layers, backend and dependency versions, configuration, dataset, command and result (Acceptance scenarios) -- A simulator pass is not native proof (Acceptance scenarios) +- A `convex-test` pass is not native backend proof (Acceptance scenarios) - A link between source and test is not a passing run (Acceptance scenarios) - A run with adjusted configuration states how it differs from production (Acceptance scenarios) - Readiness is stated plainly as specified, implemented, tested under named conditions, or operationally accepted (Acceptance scenarios) - The first experiment passes when every Layer 0, 1 and 2 scenario passes on a native backend (First experiment) - The one all-layer scenario, breaking metrics and logging and separately breaking mandatory audit, applies to every installed layer (Sc ALL-1) -- [extension] A build-tier scenario passes when a typed caller that states the change fails `tsc` before any deployment; the build tier is the fifth tier, named by Sc L2-4 alone, and needs no backend (E-14, Sc L2-4) +- [extension] A compiled-tier scenario passes when a typed caller that states the change fails `tsc` before any deployment; the compiled tier is the fifth tier, named by Sc L2-4 alone, and needs no backend (E-14, Sc L2-4) - [extension] Three statements about a scenario stay apart: it is bound when a verifier of its example resolves in the graph, it passed when a run's record holds a passing result of that verifier, and a set of scenarios is accepted only when the acceptance check finds every one of them passed in one record (E-17, Acceptance scenarios) - [extension] No example and no test stands for another scenario: a bound step names what its own test runs, and a statement about every scenario of a layer is the acceptance check's to make (E-17, Acceptance scenarios) - [extension] The acceptance check takes the rows a Spec requires and one run's record, derives its required scenarios from the graph, and passes only when every required scenario passed (E-17, First experiment) @@ -55,18 +55,18 @@ The doc's acceptance table is the contract. It lists forty-three scenario rows b The corpus maps the doc's readiness words onto SDP's rungs without collapsing them: a Spec at `defined` is specified; implemented, tested and operationally accepted are delivery facts and run evidence that the graph derives from anchors and that no author states by hand. -- tierDomain: pure tests of `decide`, `evolve` and the fold with no I/O; Sc L0-1 and L0-2 (Acceptance scenarios) -- tierSimulator: outcome combinations, serialization and classification of the four outcomes and the error codes (Acceptance scenarios, D4) -- tierNative: a disposable local backend per test, proving component and nested-mutation behavior, scheduling, contention and deployment (Acceptance scenarios, S13) -- tierEndToEnd: the production composition on a representative dataset; Sc L2-8, L2-9, L3-7, L4-3 (Acceptance scenarios) -- tierBuild: [extension] a type test of a composition's generated `api`: a reference to a name the module does not export, and a call that breaks the argument contract, each marked with the compiler's expected-error directive, so the compile fails when either stops being an error; run by the type check with no backend; Sc L2-4 (E-14, Sc L2-4) +- tierDomain: the pure test tier, tests of `decide`, `evolve` and the fold with no I/O; Sc L0-1 and L0-2 (Acceptance scenarios) +- tierSimulator: the `convex-test` tier, outcome combinations, serialization and classification of the four outcomes and the error codes (Acceptance scenarios, D4) +- tierNative: the native backend tier, a disposable local backend per test, proving component and nested-mutation behavior, scheduling, contention and deployment (Acceptance scenarios, S13) +- tierEndToEnd: the doc's end to end tier is the native backend tier on the production composition with a representative dataset; Sc L2-8, L2-9, L3-7, L4-3 (Acceptance scenarios) +- tierBuild: [extension] the compiled tier, a type test of a composition's generated `api`: a reference to a name the module does not export, and a call that breaks the argument contract, each marked with the compiler's expected-error directive, so the compile fails when either stops being an error; run by the type check with no backend; Sc L2-4 (E-14, Sc L2-4) - evidenceRecord: commit, installed layers, backend and dependency versions, configuration, dataset, command and result, plus the difference from production when configuration was adjusted (Acceptance scenarios) - readinessWords: specified, implemented, tested under named conditions, operationally accepted; never a percentage or a score (Acceptance scenarios) -- typeRequiredScenario: [extension] `interface RequiredScenario { row: string; example: string | null; tier: "domain" | "simulator" | "native" | "end to end" | "build" | null; composition: "fixture" | "production" | null; verifiers: { id: string; file: string }[] }`; the check first reads every example Spec in the graph whose opening line begins with `Sc `; any such opening line that cannot be read whole, including one whose row cannot be read, makes the check unable to answer, whether or not its row is required; after all such opening lines can be read, the check makes one entry per example whose opening line names a required row, and one entry with no example for a required row that no example names (E-17) -- requiredScenarioSource: [extension] `row`, `tier` and `composition` are read from the opening line of the example's narrative, whose parts are separated by ` · `; every example Spec in the graph whose opening line begins with `Sc ` must have a readable row, tier and composition where one is named, whether or not its row is required or can be read; after removing the full stop that ends the line, the first part is exactly the row, as in `Sc L2-3`, and the second is exactly a tier's name followed by the word `tier` and nothing else, as in `native tier`, so `native tierBROKEN` cannot be read; the third part, when it begins with `fixture` or `production`, must read exactly `fixture composition` or `production composition`, and a third part that begins otherwise, and every part after the third, describes the case; `composition` is `null` when the opening line names none, as it is for a domain, simulator or build example; `verifiers` are the example's enabled test anchors, each by its ID and its file (E-17, E-15) +- typeRequiredScenario: [extension] `interface RequiredScenario { row: string; example: string | null; tier: "compiled" | "pure test" | "convex-test" | "native backend" | null; composition: "fixture" | "production" | null; verifiers: { id: string; file: string }[] }`; the check first reads every example Spec in the graph whose opening line begins with `Sc `; any such opening line that cannot be read whole, including one whose row cannot be read, makes the check unable to answer, whether or not its row is required; after all such opening lines can be read, the check makes one entry per example whose opening line names a required row, and one entry with no example for a required row that no example names (E-17) +- requiredScenarioSource: [extension] `row`, `tier` and `composition` are read from the opening line of the example's narrative, whose parts are separated by ` · `; every example Spec in the graph whose opening line begins with `Sc ` must have a readable row, tier and composition where one is named, whether or not its row is required or can be read; after removing the full stop that ends the line, the first part is exactly the row, as in `Sc L2-3`, and the second is exactly a tier's name followed by the word `tier` and nothing else, as in `native backend tier`, so `native backend tierBROKEN` cannot be read; the third part, when it begins with `fixture` or `production`, must read exactly `fixture composition` or `production composition`, and a third part that begins otherwise, and every part after the third, describes the case; `composition` is `null` when the opening line names none, as it is for a compiled, pure test or `convex-test` example; `verifiers` are the example's enabled test anchors, each by its ID and its file (E-17, E-15) - typeAcceptanceVerdict: [extension] `interface AcceptanceVerdict { result: "passed" | "not passed"; commit: string; clean: boolean; runResult: "passed" | "failed" | "interrupted"; scenarios: (RequiredScenario & { status: "passed" | "failed" | "absent" | "missing"; reason: string | null })[] }` (E-17) -- acceptanceRecord: [extension] the record the check reads is the run record of `spec:platform.native-harness`, made by one run of every test project together, so that it holds the results of the domain and build tiers beside the native ones (E-17, E-15) -- acceptanceJoin: [extension] a verifier's results are the record's test entries whose `file` is the verifier's file; a scenario passed when each of its verifiers has at least one result, every result is `passed` and of the project that runs the scenario's tier, `pure` for domain, `simulator` for simulator, `native` for native and end to end and `types` for build, and, for a native or end-to-end scenario, the example names a composition, the results between them name at least one backend, and every backend they name has that composition (E-17, E-15, E-14) +- acceptanceRecord: [extension] the record the check reads is the run record of `spec:platform.native-harness`, made by one run of every test project together, so that it holds the results of the pure test and compiled tiers beside the native backend ones (E-17, E-15) +- acceptanceJoin: [extension] a verifier's results are the record's test entries whose `file` is the verifier's file; a scenario passed when each of its verifiers has at least one result, every result is `passed` and of the project that runs the scenario's tier, `types` for compiled, `pure` for pure test, `simulator` for `convex-test` and `native` for native backend, and, for a native backend scenario, the example names a composition, the results between them name at least one backend, and every backend they name has that composition (E-17, E-15, E-14) - acceptanceCommand: [extension] `npm run acceptance` checks the rows `acceptanceRows` of `spec:application.first-experiment` names against the newest record under `evidence/runs/` or against the record a path argument names; a candidate for the newest record is a `.json` file directly under `evidence/runs/` that parses as an object with a text `commit`, a boolean `clean`, a `result` of `passed`, `failed` or `interrupted`, a list `tests` and a text `startedAt`, and the newest is the candidate whose `startedAt` is the largest, with equal `startedAt` values resolved by choosing the file whose name sorts last; a record that cannot be parsed, that lacks those fields other than `startedAt`, or that has a test entry that is not an object with a text `name`, a text `file`, a `result` of `passed`, `failed` or `skipped`, a `project` that is absent or text and a list `backends` of objects whose `composition` is absent, `null` or text, is read as no record, and no older record is tried; it prints one line per required scenario with its row, example, tier, composition, verifier files, status and reason, and then one line, `acceptance: passed` or `acceptance: not passed`, with the count of each status, the run's result, whether the tree was clean and the record's commit (E-17) - acceptanceExit: [extension] the command exits 0 when the check passes; 1 when a required scenario is failed or absent, the record's run did not pass or its tree was not clean; 3 when none of those holds and a required scenario is missing; and 2 when it cannot answer, which is no record, a graph that does not derive, a Spec that names no required row, an example Spec in the graph whose opening line begins with `Sc ` and cannot be read whole, including one whose row cannot be read, whether or not its row is required, or any other failure to answer; for every failure to answer it prints the one line `acceptance: cannot answer` with the reason and exits 2, so a failure to answer never ends without that line (E-17) diff --git a/design/specs/platform/native-harness.admin-key-acting-as-identity-reaches-internal-function.sdp.md b/design/specs/platform/native-harness.admin-key-acting-as-identity-reaches-internal-function.sdp.md index 6febcab..054a4b8 100644 --- a/design/specs/platform/native-harness.admin-key-acting-as-identity-reaches-internal-function.sdp.md +++ b/design/specs/platform/native-harness.admin-key-acting-as-identity-reaches-internal-function.sdp.md @@ -9,7 +9,7 @@ relations: --- # The admin key acting as an identity reaches an internal function -E-13 · native tier · fixture composition. +E-13 · native backend tier · fixture composition. ## Intent @@ -25,6 +25,6 @@ And the table the internal mutation writes holds {rows: 1} row ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that a public query called by the same client returns the identity it acts as, so the client is indistinguishable from an ordinary caller inside a handler. - This is the one test in which a caller holds the admin key. diff --git a/design/specs/platform/native-harness.fixture-issuer-token-yields-identity.sdp.md b/design/specs/platform/native-harness.fixture-issuer-token-yields-identity.sdp.md index 7b418e2..202003c 100644 --- a/design/specs/platform/native-harness.fixture-issuer-token-yields-identity.sdp.md +++ b/design/specs/platform/native-harness.fixture-issuer-token-yields-identity.sdp.md @@ -9,7 +9,7 @@ relations: --- # A token signed by the fixture issuer yields the caller's identity -E-13 · native tier · fixture composition. +E-13 · native backend tier · fixture composition. ## Intent @@ -25,5 +25,5 @@ And the same query from a client with no token returns no identity ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The test asserts that `tokenIdentifier` is the issuer and the subject joined by a vertical bar, and that a token signed by a key outside the key set is refused. diff --git a/design/specs/platform/native-harness.ordinary-client-refused-component-function.sdp.md b/design/specs/platform/native-harness.ordinary-client-refused-component-function.sdp.md index 775f2f8..7745ac3 100644 --- a/design/specs/platform/native-harness.ordinary-client-refused-component-function.sdp.md +++ b/design/specs/platform/native-harness.ordinary-client-refused-component-function.sdp.md @@ -9,7 +9,7 @@ relations: --- # An ordinary client cannot call a component's function -E-13, E-15 · native tier · fixture composition. +E-13, E-15 · native backend tier · fixture composition. ## Intent @@ -28,7 +28,7 @@ And the same mutation called with the harness's admin access writes {rowsAfterAd ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - Each HTTP refusal is identified by the text of Convex's own error, so a call that fails for another reason does not pass. - On the public route the test checks that the refused call named the same function the admin call runs, so a call to another name does not pass. - `BadDeployKey` is the gate of the endpoint that takes a component path: it answers every caller without the admin key, whatever the path, so on that route the error says nothing about the function, and the admin call is what shows that the function exists. diff --git a/design/specs/platform/native-harness.ordinary-client-refused-internal-function.sdp.md b/design/specs/platform/native-harness.ordinary-client-refused-internal-function.sdp.md index 9a504b3..2ff8e75 100644 --- a/design/specs/platform/native-harness.ordinary-client-refused-internal-function.sdp.md +++ b/design/specs/platform/native-harness.ordinary-client-refused-internal-function.sdp.md @@ -9,7 +9,7 @@ relations: --- # An ordinary client cannot call an internal function -E-13, E-15 · native tier · fixture composition. +E-13, E-15 · native backend tier · fixture composition. ## Intent @@ -26,7 +26,7 @@ And the same mutation called with the harness's admin access writes {rowsAfterAd ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The refusal is identified by the text of Convex's own error, so a call that fails for another reason does not pass. - The test repeats the refused call from a client with no token and asserts the same error, and reads the table with the harness's admin access. - The admin call comes last and on the same deployment, so the refused function is shown to exist and to write its row. diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 84f6bb7..abaa94d 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -49,12 +49,12 @@ The doc says that native tests prove component and nested-mutation behavior, sch - [extension] What a command read and wrote, in documents and bytes, is taken from the `usageStats` of its top-level completion record, never from a function that exists for a test (E-15, First experiment) - [extension] Where a scenario asserts that something did not run, the test shows the refusal's own error, shows that nothing the thing writes is stored, and shows on the same deployment that the thing exists and runs, through a control that an ordinary client sends where a client may call the thing and that is sent with admin access otherwise; where no error tells a refusal from a failure, the fixture composition supplies a part that fails the call if it runs (E-15) - [extension] An error text, a count or a wait that an example of this harness binds is what the pinned backend release answers; another answer on a new release is a finding to rule on, and the example is rebound, not the test repaired to keep the old value (E-15) -- [extension] The native tier controls a run through four things: the backend process, its clients, the deployment's environment variables and, on the fixture composition, fixture functions (E-15) +- [extension] The native backend tier controls a run through four things: the backend process, its clients, the deployment's environment variables and, on the fixture composition, fixture functions (E-15) - [extension] The harness starts the backend on fresh storage, stops it, kills it, and restarts it on the same storage (E-15, S13) - [extension] The harness registers the cleanup of a backend before the start of that backend can stall; a run that ends or is interrupted kills every backend process it started and removes every backend directory (E-15) - [extension] Before the harness signals a process it reads from the process table the instance name and the storage path it gave that backend, and it signals no process that does not show both; a backend whose process it cannot inspect keeps its directory and its ownership record, and the run fails naming it (E-15) - [extension] A client under test may close, and may lose a response (E-15) -- [extension] The native tier cannot move the backend's clock and cannot switch its scheduler off (E-15, S13) +- [extension] The native backend tier cannot move the backend's clock and cannot switch its scheduler off (E-15, S13) - [extension] A scenario that needs time to pass waits in real time or shortens a duration that its owning Spec makes configuration, and a run with a shortened duration is adjusted configuration (E-15, Acceptance scenarios) - [extension] A native run's record names the backend that ran by the hash of its executable and, when that hash is the pinned one, by its release; it names the identity source, and the commit with whether the tree was clean (E-15, Acceptance scenarios) - [extension] A run's record takes its facts from the backends that ran, and a measurement is stored on the test that recorded it (E-15, Acceptance scenarios) diff --git a/design/specs/processes/approvals.approval-races.sdp.md b/design/specs/processes/approvals.approval-races.sdp.md index c883d1b..b34f14d 100644 --- a/design/specs/processes/approvals.approval-races.sdp.md +++ b/design/specs/processes/approvals.approval-races.sdp.md @@ -9,7 +9,7 @@ relations: --- # Race approval, expiry, revocation and execution -Sc L4-2 · native tier. +Sc L4-2 · native backend tier. ## Intent @@ -25,6 +25,6 @@ And every proposal ends approved and executed, rejected, or expired, with no pro ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test enumerates the interleavings of approve, expire, revoke and execute and asserts each ends in one explicit transition, that an execution after revocation is refused by the current grant check, and that a changed input hash voids the approval (D16, Law 5). - The test asserts concurrent executions of one approved proposal commit once, because execution consumes the approval (D16, D1). diff --git a/design/specs/processes/workflow-processes.deploy-with-process-in-flight.sdp.md b/design/specs/processes/workflow-processes.deploy-with-process-in-flight.sdp.md index 2e30296..9ea9eb4 100644 --- a/design/specs/processes/workflow-processes.deploy-with-process-in-flight.sdp.md +++ b/design/specs/processes/workflow-processes.deploy-with-process-in-flight.sdp.md @@ -9,7 +9,7 @@ relations: --- # Deploy while an old process is in flight -Sc L4-3 · end to end tier. +Sc L4-3 · native backend tier · production composition. ## Intent @@ -25,6 +25,6 @@ And the process record shows which of the three happened and never shows complet ## Verification — executable -- Runs in the end to end tier; every test owns its disposable backend. +- Runs in the native backend tier on the production composition; every test owns its disposable backend. - The test deploys a release that keeps the old definition registered and asserts the old run continues; deploys one that removes it and asserts the process record is blocked with a remedy and the run's determinism violation is recorded, never a silent failure (D16, S11). - The test asserts a blocked process never reports success and that its payment obligation, if any, keeps its own state (Law 7, D15). diff --git a/design/specs/processes/workflow-processes.restart-after-external-step.sdp.md b/design/specs/processes/workflow-processes.restart-after-external-step.sdp.md index c8dfd40..5a31360 100644 --- a/design/specs/processes/workflow-processes.restart-after-external-step.sdp.md +++ b/design/specs/processes/workflow-processes.restart-after-external-step.sdp.md @@ -9,7 +9,7 @@ relations: --- # Restart a process after an external step completed -Sc L4-1 · native tier. +Sc L4-1 · native backend tier. ## Intent @@ -25,6 +25,6 @@ And the process continues from the confirmation step with the obligation's evide ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. +- Runs in the native backend tier; every test owns its disposable backend. - The test interrupts the run after the obligation settled and before the workflow journaled the wait's result, restarts the run, and asserts the provider stub recorded one effect and the obligation has one attempt (D16, D15). - The test asserts the process record's status moved from awaiting payment to confirmed exactly once (D16, Law 7). diff --git a/fixture/convex/rejectionCommands.ts b/fixture/convex/rejectionCommands.ts index 0b4aa0b..1460baf 100644 --- a/fixture/convex/rejectionCommands.ts +++ b/fixture/convex/rejectionCommands.ts @@ -11,7 +11,7 @@ import { listDetailsTitle } from "./depot/streams.js"; const { operations } = components.depot; const writesDocument = [{ contextId: "depot", streamType: "document" }]; // Its executor throws a rejection in the wire shape that is invalid twice over: its code is one the -// declaration does not list, and it names another command. +// declaration does not list, and it names another entry. const createRejectedDocumentDeclaration: CommandDeclaration< { documentId: string }, null @@ -33,7 +33,7 @@ const createRejectedDocumentDeclaration: CommandDeclaration< throw new ConvexError({ kind: "rejection", code: "notDeclared", - commandType: "SomeOtherCommand", + entry: "SomeOtherCommand", message: "x", }); }, diff --git a/harness/acceptance.ts b/harness/acceptance.ts index 129425c..3f13165 100644 --- a/harness/acceptance.ts +++ b/harness/acceptance.ts @@ -16,8 +16,10 @@ const anchor = codeAnchor({ }); void anchor; +// The tiers an example's opening line names. The doc's end to end tier is no tier of its own: an +// example of it names the native backend tier and the production composition. export type AcceptanceTier = - "domain" | "simulator" | "native" | "end to end" | "build"; + "compiled" | "pure test" | "convex-test" | "native backend"; export type AcceptanceComposition = "fixture" | "production"; export interface RequiredScenario { row: string; @@ -75,22 +77,21 @@ export interface AcceptanceRecord { export class AcceptanceUnanswerable extends Error {} const tiers: readonly AcceptanceTier[] = [ - "domain", - "simulator", - "native", - "end to end", - "build", + "compiled", + "pure test", + "convex-test", + "native backend", ]; const compositions: readonly AcceptanceComposition[] = [ "fixture", "production", ]; +// The Vitest project that runs each tier. const projectOfTier: Record = { - domain: "pure", - simulator: "simulator", - native: "native", - "end to end": "native", - build: "types", + compiled: "types", + "pure test": "pure", + "convex-test": "simulator", + "native backend": "native", }; const rowPattern = /\bSc [A-Z]+\d*-\d+\b/g; @@ -224,10 +225,7 @@ function judge( `${test.file} ran in the ${test.project ?? "unnamed"} project, not ${project}`, ); } - if ( - results.length > 0 && - (scenario.tier === "native" || scenario.tier === "end to end") - ) { + if (results.length > 0 && scenario.tier === "native backend") { const backends = results.flatMap((test) => test.backends); if (scenario.composition === null) failures.push("the example names no composition"); diff --git a/src/command/authority.ts b/src/command/authority.ts index 55febd5..6031f5c 100644 --- a/src/command/authority.ts +++ b/src/command/authority.ts @@ -181,7 +181,7 @@ export type QueryPolicy = { const noServiceIssuers: ReadonlySet = new Set(); // Establishes the actor and authorizes it for the tenant, as the pipeline's steps 2 and 4 do, and // returns the actor. A refusal is the rejection a command throws for the same refusal, with the -// query's name where a command's type goes. It writes nothing. +// query's name as the entry that refused. It writes nothing. export async function authorizeQuery( parentCtx: GenericQueryCtx, policy: QueryPolicy, @@ -193,7 +193,7 @@ export async function authorizeQuery( if (actor === null) reject({ code: "unauthenticated", - commandType: policy.name, + entry: policy.name, message: `${policy.name} needs an authenticated caller`, }); const decision = await authorize(ctx, { @@ -205,7 +205,7 @@ export async function authorizeQuery( if (!decision.allowed) reject({ code: "forbidden", - commandType: policy.name, + entry: policy.name, message: `The caller may not read ${policy.name} in this tenant`, details: { reason: decision.reason }, }); diff --git a/src/command/declaration.ts b/src/command/declaration.ts index 6b9f9b0..4c96b21 100644 --- a/src/command/declaration.ts +++ b/src/command/declaration.ts @@ -115,7 +115,7 @@ export function publicCommand( if (actor === null) reject({ code: "unauthenticated", - commandType: decl.name, + entry: decl.name, message: `${decl.name} needs an authenticated caller`, }); return await runPipeline(ctx, decl, { diff --git a/src/command/outcome-boundary.ts b/src/command/outcome-boundary.ts index 5574cb8..5681ee3 100644 --- a/src/command/outcome-boundary.ts +++ b/src/command/outcome-boundary.ts @@ -32,7 +32,8 @@ export type TransientCode = "rateLimited" | "capacity" | "writePaused"; export type RejectionData = Rejection & { kind: "rejection"; code: RejectionCode; - commandType: string; + // The name of the entry that refused. + entry: string; }; export type TransientData = { kind: "transient"; @@ -45,7 +46,7 @@ const wireRejectionValidator = v.object({ kind: v.literal("rejection"), code: v.string(), message: v.string(), - commandType: v.string(), + entry: v.string(), details: v.optional(v.record(v.string(), v.any())), }); export const errorDataValidator = v.union( @@ -99,27 +100,27 @@ function fitsShape({ form, data }: Candidate): boolean { // The bound on a rejection's details, measured as Convex measures a value. const limitDetailsBytes = 16384; // Throws a plain Error, a technical failure, when the details measure above the bound. -function checkDetails(rejection: Rejection, commandType: string) { +function checkDetails(rejection: Rejection, entry: string) { if (rejection.details === undefined) return; const bytes = getConvexSize(rejection.details); if (bytes > limitDetailsBytes) throw new Error( - `A ${rejection.code} rejection of ${commandType} carries ${bytes} bytes of details, above ${limitDetailsBytes}`, + `A ${rejection.code} rejection of ${entry} carries ${bytes} bytes of details, above ${limitDetailsBytes}`, ); } export function reject(data: Omit, "kind">): never { - checkDetails(data, data.commandType); + checkDetails(data, data.entry); throw new ConvexError({ kind: "rejection", ...data }); } export function refuseTransient(data: Omit): never { throw new ConvexError({ kind: "transient", ...data }); } -// Every rejection candidate passes its shape, the declaration's code list, its command name and the +// Every rejection candidate passes its shape, the declaration's code list, the entry it names and the // details bound. A bare rejection gains the wire shape; a valid rejection already in that shape keeps // its identity. Every other throw passes unchanged. export function normalizeThrown( error: unknown, - commandType: string, + entry: string, rejections: readonly string[], ): never { const candidate = @@ -128,27 +129,27 @@ export function normalizeThrown( const wire = candidate.form === "wire"; if (!fitsShape(candidate)) throw new Error( - `${commandType} received a rejection that does not fit the ${candidate.form} shape`, + `${entry} received a rejection that does not fit the ${candidate.form} shape`, ); - // fitsShape established code, message and details for both forms, and commandType for the wire form. - const rejection = candidate.data as Rejection & { commandType?: string }; + // fitsShape established code, message and details for both forms, and entry for the wire form. + const rejection = candidate.data as Rejection & { entry?: string }; const { code } = rejection; if ( !Object.hasOwn(platformRejectionCodes, code) && !rejections.includes(code) ) throw new Error( - `${commandType} was rejected with the code ${code}, which is neither a platform code nor one its declaration lists in rejections`, + `${entry} was rejected with the code ${code}, which is neither a platform code nor one its declaration lists in rejections`, ); - if (wire && rejection.commandType !== commandType) + if (wire && rejection.entry !== entry) throw new Error( - `${commandType} received a rejection naming ${String(rejection.commandType)}`, + `${entry} received a rejection naming ${String(rejection.entry)}`, ); - checkDetails(rejection, commandType); + checkDetails(rejection, entry); if (wire) throw error; throw new ConvexError({ kind: "rejection", - commandType, + entry, ...rejection, }); } diff --git a/src/command/pipeline.ts b/src/command/pipeline.ts index 8741551..3cd18b0 100644 --- a/src/command/pipeline.ts +++ b/src/command/pipeline.ts @@ -153,7 +153,7 @@ function parse(decl: CommandDeclaration, call: PipelineCall) { if (length > limit) reject({ code: "invalidInput", - commandType, + entry: decl.name, message: `${field} has at most ${limit} bytes of UTF-8`, details: { field, length, limit }, }); @@ -169,7 +169,7 @@ function parse(decl: CommandDeclaration, call: PipelineCall) { if (maxItems !== undefined && items > maxItems) reject({ code: "operationTooLarge", - commandType, + entry: decl.name, message: `${commandType} takes at most ${maxItems} items, not ${items}`, details: { items, maxItems }, }); @@ -177,13 +177,13 @@ function parse(decl: CommandDeclaration, call: PipelineCall) { if (maxBytes !== undefined && bytes > maxBytes) reject({ code: "operationTooLarge", - commandType, + entry: decl.name, message: `${commandType} takes at most ${maxBytes} bytes of input, not ${bytes}`, details: { bytes, maxBytes }, }); const refused = decl.refine?.(call.input) ?? null; if (refused !== null) - reject({ ...refused, code: "invalidInput", commandType }); + reject({ ...refused, code: "invalidInput", entry: decl.name }); } // Steps 1 and 3 to 11 for either entry; the public entry ran step 2 before it built the call. Step 11 // emits no diagnostic. @@ -208,7 +208,7 @@ export async function runPipeline( if (!decision.allowed) reject({ code: "forbidden", - commandType, + entry: decl.name, message: `The caller may not run ${commandType} in this tenant`, details: { reason: decision.reason }, }); @@ -237,13 +237,13 @@ export async function runPipeline( case "unsupportedVersion": reject({ code: "unsupportedContractVersion", - commandType, + entry: decl.name, message: "This request key was used under another contract version", }); case "conflict": reject({ code: "idempotencyConflict", - commandType, + entry: decl.name, message: "This request key was used with other input", }); case "duplicate": { diff --git a/tests/native/cancel-order-steps.ts b/tests/native/cancel-order-steps.ts index 4f10b4a..24a40b4 100644 --- a/tests/native/cancel-order-steps.ts +++ b/tests/native/cancel-order-steps.ts @@ -180,7 +180,7 @@ export function callerReceives( expect(world.error).toBeInstanceOf(ConvexError); expect(rejectionData(world)).toMatchObject({ kind: "rejection", - commandType: "CancelOrder", + entry: "CancelOrder", }); return; } diff --git a/tests/native/command-without-grant-forbidden.test.ts b/tests/native/command-without-grant-forbidden.test.ts index 3d522af..d3e22ae 100644 --- a/tests/native/command-without-grant-forbidden.test.ts +++ b/tests/native/command-without-grant-forbidden.test.ts @@ -25,7 +25,7 @@ test("native: an ordinary client without a grant is refused with forbidden, and expect((error as ConvexError).data).toEqual({ kind: "rejection", code: "forbidden", - commandType: "CreateDocument", + entry: "CreateDocument", message: "The caller may not run CreateDocument in this tenant", details: { reason: "no_grant" }, }); diff --git a/tests/native/context-queries.test.ts b/tests/native/context-queries.test.ts index 95dab11..b4224ce 100644 --- a/tests/native/context-queries.test.ts +++ b/tests/native/context-queries.test.ts @@ -312,10 +312,10 @@ test("native: each relay refuses a caller with no identity and one with no grant documentId: "doc-1", }), ), - ).toMatchObject({ kind: "rejection", code, commandType: "getDocument" }); + ).toMatchObject({ kind: "rejection", code, entry: "getDocument" }); expect( await refusal(read(caller, { cursor: null, numItems: 10 })), - ).toMatchObject({ kind: "rejection", code, commandType: "listDocuments" }); + ).toMatchObject({ kind: "rejection", code, entry: "listDocuments" }); } // A grant for doc-1 reads doc-1 and is refused doc-2; a tenant-wide grant reads doc-2. await grant("one-document", "doc-1"); @@ -336,7 +336,7 @@ test("native: each relay refuses a caller with no identity and one with no grant ), ).toMatchObject({ code: "forbidden", - commandType: "getDocument", + entry: "getDocument", details: { reason: "subject_mismatch" }, }); expect( diff --git a/tests/native/first-experiment-steps.ts b/tests/native/first-experiment-steps.ts index a8b1254..c6b0da0 100644 --- a/tests/native/first-experiment-steps.ts +++ b/tests/native/first-experiment-steps.ts @@ -281,7 +281,7 @@ export function projectionJobsAre(world: ExperimentWorld, jobs: number) { expect(placed.summaryKeys).toContain(placed.orderId); } } -// The calls are counted at the pure tier: the backend's log holds no record of a component call +// The calls are counted at the pure test tier: the backend's log holds no record of a component call // inside a mutation. The executor runs on the input the test sent, against a ctx whose runMutation // records each call by its function reference and answers a canned applied outcome. export async function callsPerContextAre( @@ -533,12 +533,12 @@ export function stockItemIdRejectionIs( line: number, length: number, limit: number, - commandType: "PlaceOrder" | "ReceiveStock" = "PlaceOrder", + entry: "PlaceOrder" | "ReceiveStock" = "PlaceOrder", ) { expect((world.error as ConvexError).data).toEqual({ kind: "rejection", code: "invalidInput", - commandType, + entry, message: `Line ${line} needs a stock item ID of at most ${limit} bytes of UTF-8, not ${length}`, details: { line, length, limit }, }); diff --git a/tests/native/first-experiment.native-acceptance-production-configuration.test.ts b/tests/native/first-experiment.native-acceptance-production-configuration.test.ts index 9c52fdc..aaaf6d9 100644 --- a/tests/native/first-experiment.native-acceptance-production-configuration.test.ts +++ b/tests/native/first-experiment.native-acceptance-production-configuration.test.ts @@ -169,12 +169,12 @@ function authorityHolds(world: AcceptanceWorld) { expect(dataOf(thrown(answers, "placeOrder short of stock"))).toMatchObject({ kind: "rejection", code: "insufficientStock", - commandType: "PlaceOrder", + entry: "PlaceOrder", }); expect(dataOf(thrown(answers, "placeOrder without a grant"))).toMatchObject({ kind: "rejection", code: "forbidden", - commandType: "PlaceOrder", + entry: "PlaceOrder", }); expect(returned(answers, "getOrder")).toMatchObject({ orderId: "order-1", @@ -182,12 +182,12 @@ function authorityHolds(world: AcceptanceWorld) { total: 300, }); expect(dataOf(thrown(answers, "getOrder without an identity"))).toMatchObject( - { kind: "rejection", code: "unauthenticated", commandType: "getOrder" }, + { kind: "rejection", code: "unauthenticated", entry: "getOrder" }, ); expect(dataOf(thrown(answers, "getOrder without a grant"))).toMatchObject({ kind: "rejection", code: "forbidden", - commandType: "getOrder", + entry: "getOrder", }); expect(returned(answers, "listOrders")).toMatchObject({ page: [{ orderId: "order-1" }], diff --git a/tests/native/first-experiment.order-of-max-lines.test.ts b/tests/native/first-experiment.order-of-max-lines.test.ts index b124c39..28c59f2 100644 --- a/tests/native/first-experiment.order-of-max-lines.test.ts +++ b/tests/native/first-experiment.order-of-max-lines.test.ts @@ -109,7 +109,7 @@ bindExample( expect((error as ConvexError).data).toEqual({ kind: "rejection", code: "operationTooLarge", - commandType: "PlaceOrder", + entry: "PlaceOrder", message: "PlaceOrder takes at most 100 items, not 101", details: { items: 101, maxItems: 100 }, }); diff --git a/tests/native/idempotency-and-receipts.ui-double-submit.test.ts b/tests/native/idempotency-and-receipts.ui-double-submit.test.ts index 7d5d1f8..8028007 100644 --- a/tests/native/idempotency-and-receipts.ui-double-submit.test.ts +++ b/tests/native/idempotency-and-receipts.ui-double-submit.test.ts @@ -159,7 +159,7 @@ bindExample( expect(rejectionData(answers[1])).toMatchObject({ kind: "rejection", code: "entityExists", - commandType: "PlaceOrder", + entry: "PlaceOrder", details: { existing: entityId, current: 1 }, }); // FailIfDecided's decide fails the call if it is reached. At expected version 0 for the same @@ -174,7 +174,7 @@ bindExample( expect(rejectionData(atZero)).toMatchObject({ kind: "rejection", code: "entityExists", - commandType: "FailIfDecided", + entry: "FailIfDecided", details: { existing: entityId, current: 1 }, }); const atCurrent = await answerOf( @@ -256,7 +256,7 @@ bindExample( expect(rejectionData(another)).toMatchObject({ kind: "rejection", code: "insufficientStock", - commandType: "PlaceOrder", + entry: "PlaceOrder", }); const after = await storedNow(backend); expect(streamRow(after, "stock", "p-2")?.state).toEqual({ onHand: 0 }); diff --git a/tests/native/order-read-steps.ts b/tests/native/order-read-steps.ts index 9d87941..a2f2d2d 100644 --- a/tests/native/order-read-steps.ts +++ b/tests/native/order-read-steps.ts @@ -92,7 +92,7 @@ export function readThrows(world: ReadWorld, code: string) { expect((error as ConvexError).data).toMatchObject({ kind: "rejection", code, - commandType: "getOrder", + entry: "getOrder", }); } expect((errors[0] as ConvexError).data).toEqual( @@ -100,13 +100,13 @@ export function readThrows(world: ReadWorld, code: string) { ? { kind: "rejection", code, - commandType: "getOrder", + entry: "getOrder", message: "getOrder needs an authenticated caller", } : { kind: "rejection", code, - commandType: "getOrder", + entry: "getOrder", message: "The caller may not read getOrder in this tenant", details: { reason: "no_grant" }, }, diff --git a/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts b/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts index 30e2d5b..ede5224 100644 --- a/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts +++ b/tests/native/orders-inventory-example.cancel-second-context-rejects.test.ts @@ -31,11 +31,11 @@ bindExample( (): CancelWorld => ({}), steps, async (world) => { - // The call order and the quantities' source are shown at the pure tier, by the executor itself. + // The call order and the quantities' source are shown at the pure test tier, by the executor itself. expect(rejectionData(world)).toEqual({ kind: "rejection", code: "insufficientAllocation", - commandType: "CancelOrder", + entry: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 }, }); diff --git a/tests/native/orders-inventory-example.cancel-twice-rejected.test.ts b/tests/native/orders-inventory-example.cancel-twice-rejected.test.ts index 25d0af6..c648370 100644 --- a/tests/native/orders-inventory-example.cancel-twice-rejected.test.ts +++ b/tests/native/orders-inventory-example.cancel-twice-rejected.test.ts @@ -31,7 +31,7 @@ bindExample( expect(rejectionData(world)).toEqual({ kind: "rejection", code: "orderAlreadyCancelled", - commandType: "CancelOrder", + entry: "CancelOrder", message: "The order is already cancelled", }); const { backend } = required(world.order, "the backend"); diff --git a/tests/native/orders-inventory-example.cancel-unknown-order-rejected.test.ts b/tests/native/orders-inventory-example.cancel-unknown-order-rejected.test.ts index 383c89b..690929b 100644 --- a/tests/native/orders-inventory-example.cancel-unknown-order-rejected.test.ts +++ b/tests/native/orders-inventory-example.cancel-unknown-order-rejected.test.ts @@ -34,7 +34,7 @@ bindExample( expect(rejectionData(world)).toEqual({ kind: "rejection", code: "orderNotFound", - commandType: "CancelOrder", + entry: "CancelOrder", message: "The order does not exist", }); const order = required(world.order, "the backend"); diff --git a/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts b/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts index 0a34cdb..740347d 100644 --- a/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts +++ b/tests/native/orders-inventory-example.cancel-without-grant-refused.test.ts @@ -30,7 +30,7 @@ void anchor; const forbidden = { kind: "rejection", code: "forbidden", - commandType: "CancelOrder", + entry: "CancelOrder", message: "The caller may not run CancelOrder in this tenant", details: { reason: "no_grant" }, }; diff --git a/tests/native/outcome-boundary.rejected-then-retried-after-change.test.ts b/tests/native/outcome-boundary.rejected-then-retried-after-change.test.ts index d4a0075..1399bc1 100644 --- a/tests/native/outcome-boundary.rejected-then-retried-after-change.test.ts +++ b/tests/native/outcome-boundary.rejected-then-retried-after-change.test.ts @@ -208,7 +208,7 @@ bindExample( expect((world.firstError as ConvexError).data).toMatchObject({ kind: "rejection", code: "insufficientStock", - commandType: "ClaimStock", + entry: "ClaimStock", }); expect(world.receiptsAfterFirstRun).toEqual([]); const response = retried(world); diff --git a/tests/native/parent-use-cases.second-context-rejects.test.ts b/tests/native/parent-use-cases.second-context-rejects.test.ts index b41e9aa..227156c 100644 --- a/tests/native/parent-use-cases.second-context-rejects.test.ts +++ b/tests/native/parent-use-cases.second-context-rejects.test.ts @@ -102,7 +102,7 @@ bindExample(contract, (): World => ({}), { expect((world.error as ConvexError).data).toEqual({ kind: "rejection", code: "insufficientStock", - commandType: "PlaceOrder", + entry: "PlaceOrder", message: "Cannot allocate 3 when 2 are available", details: { requested: 3, available: 2 }, }); diff --git a/tests/native/production-orders.test.ts b/tests/native/production-orders.test.ts index 04cdbb8..40ffab1 100644 --- a/tests/native/production-orders.test.ts +++ b/tests/native/production-orders.test.ts @@ -280,7 +280,7 @@ test("native: an order for stock that is not there is rejected insufficientStock expect((error as ConvexError).data).toEqual({ kind: "rejection", code: "insufficientStock", - commandType: "PlaceOrder", + entry: "PlaceOrder", message: "Cannot allocate 1 when 0 are available", details: { requested: 1, available: 0 }, }); diff --git a/tests/native/read-models.test.ts b/tests/native/read-models.test.ts index ff1b37d..8a80bc4 100644 --- a/tests/native/read-models.test.ts +++ b/tests/native/read-models.test.ts @@ -207,13 +207,13 @@ test("native: a refused parent query throws the rejection shape to an HTTP clien expect((anonymous as ConvexError).data).toEqual({ kind: "rejection", code: "unauthenticated", - commandType: "listDocumentSummaries", + entry: "listDocumentSummaries", message: "listDocumentSummaries needs an authenticated caller", }); expect((stranger as ConvexError).data).toEqual({ kind: "rejection", code: "forbidden", - commandType: "listDocumentSummaries", + entry: "listDocumentSummaries", message: "The caller may not read listDocumentSummaries in this tenant", details: { reason: "no_grant" }, }); @@ -231,7 +231,7 @@ test("native: a refused parent query throws the rejection shape to an HTTP clien expect(subscribed).toBeInstanceOf(ConvexError); expect((subscribed as ConvexError).data).toMatchObject({ code: "unauthenticated", - commandType: "listDocumentSummaries", + entry: "listDocumentSummaries", }); // The command was sent with no request key, and a read writes nothing: no receipt exists. expect(await backend.admin.readTable("receipts")).toEqual([]); diff --git a/tests/native/receipted-calls.ts b/tests/native/receipted-calls.ts index b837380..30e365c 100644 --- a/tests/native/receipted-calls.ts +++ b/tests/native/receipted-calls.ts @@ -325,7 +325,7 @@ export function thenFirst(world: ReceiptWorld, { first }: { first: string }) { expect(data).toMatchObject({ kind: "rejection", code: "idempotencyConflict", - commandType: world.commandType, + entry: world.commandType, }); expect(data).not.toHaveProperty("details"); expect(JSON.stringify(data)).not.toContain( diff --git a/tests/native/tenancy-and-authority.revoked-then-retried.test.ts b/tests/native/tenancy-and-authority.revoked-then-retried.test.ts index 5dbb180..11ca56b 100644 --- a/tests/native/tenancy-and-authority.revoked-then-retried.test.ts +++ b/tests/native/tenancy-and-authority.revoked-then-retried.test.ts @@ -58,7 +58,7 @@ bindExample( expect((error as ConvexError).data).toEqual({ kind: "rejection", code: "forbidden", - commandType: "CreateDocument", + entry: "CreateDocument", message: "The caller may not run CreateDocument in this tenant", details: { reason: "no_grant" }, }); diff --git a/tests/native/tenancy-steps.ts b/tests/native/tenancy-steps.ts index 181ccfd..d2fbaac 100644 --- a/tests/native/tenancy-steps.ts +++ b/tests/native/tenancy-steps.ts @@ -320,7 +320,7 @@ export function answerIs( expect(data).toMatchObject({ kind: "rejection", code: answer === "conflict" ? "idempotencyConflict" : "forbidden", - commandType: "CreateDocument", + entry: "CreateDocument", }); // A conflict carries nothing of the stored receipt. if (answer === "conflict") expect(data).not.toHaveProperty("details"); diff --git a/tests/pure/acceptance.test.ts b/tests/pure/acceptance.test.ts index 88dc061..a1e3296 100644 --- a/tests/pure/acceptance.test.ts +++ b/tests/pure/acceptance.test.ts @@ -41,17 +41,17 @@ function sampleGraph( examples: SampleExample[] = [ { id: "spec:sample.evaluate", - opening: "Sc L0-1 · domain tier.", + opening: "Sc L0-1 · pure test tier.", files: ["tests/pure/evaluate.test.ts"], }, { id: "spec:sample.transition", - opening: "Sc L1-1 · native tier · fixture composition.", + opening: "Sc L1-1 · native backend tier · fixture composition.", files: ["tests/native/transition.test.ts"], }, { id: "spec:sample.production", - opening: "Sc L2-9 · end to end tier · production composition.", + opening: "Sc L2-9 · native backend tier · production composition.", files: ["tests/native/production.test.ts"], }, ], @@ -164,17 +164,17 @@ test("pure: a required scenario whose verifier is removed from the graph is miss const graph = sampleGraph([ { id: "spec:sample.evaluate", - opening: "Sc L0-1 · domain tier.", + opening: "Sc L0-1 · pure test tier.", files: ["tests/pure/evaluate.test.ts"], }, { id: "spec:sample.transition", - opening: "Sc L1-1 · native tier · fixture composition.", + opening: "Sc L1-1 · native backend tier · fixture composition.", files: [], }, { id: "spec:sample.production", - opening: "Sc L2-9 · end to end tier · production composition.", + opening: "Sc L2-9 · native backend tier · production composition.", files: ["tests/native/production.test.ts"], }, ]); @@ -208,7 +208,7 @@ test("pure: a file with two tests passes only when both passed", () => { expect(statusOf(record, "Sc L1-1")).toBe("failed"); }); -test("pure: a native scenario fails on another composition, on no backend, or when its example names no composition", () => { +test("pure: a native backend scenario fails on another composition, on no backend, or when its example names no composition", () => { const wrong = sampleRecord(); wrong.tests[2]!.backends = [{ composition: "fixture" }]; expect(statusOf(wrong, "Sc L2-9")).toBe("failed"); @@ -224,17 +224,17 @@ test("pure: a native scenario fails on another composition, on no backend, or wh const unnamed = sampleGraph([ { id: "spec:sample.evaluate", - opening: "Sc L0-1 · domain tier.", + opening: "Sc L0-1 · pure test tier.", files: ["tests/pure/evaluate.test.ts"], }, { id: "spec:sample.transition", - opening: "Sc L1-1 · native tier.", + opening: "Sc L1-1 · native backend tier.", files: ["tests/native/transition.test.ts"], }, { id: "spec:sample.production", - opening: "Sc L2-9 · end to end tier · production composition.", + opening: "Sc L2-9 · native backend tier · production composition.", files: ["tests/native/production.test.ts"], }, ]); @@ -252,17 +252,17 @@ test("pure: a native scenario fails on another composition, on no backend, or wh }); }); -test("pure: a native scenario whose only result came from the pure project is failed", () => { +test("pure: a native backend scenario whose only result came from the pure project is failed", () => { const record = sampleRecord(); record.tests[1]!.project = "pure"; expect(statusOf(record, "Sc L1-1")).toBe("failed"); const unnamed = sampleRecord(); delete unnamed.tests[1]!.project; expect(statusOf(unnamed, "Sc L1-1")).toBe("failed"); - // A domain scenario does not pass on a native result either. - const domain = sampleRecord(); - domain.tests[0]!.project = "native"; - expect(statusOf(domain, "Sc L0-1")).toBe("failed"); + // A pure test scenario does not pass on a result of the native project either. + const pure = sampleRecord(); + pure.tests[0]!.project = "native"; + expect(statusOf(pure, "Sc L0-1")).toBe("failed"); }); test("pure: every scenario passed, and a run that failed or a tree that was not clean still fails the check with exit 1", async () => { @@ -375,7 +375,7 @@ test("pure: a scenario with one verifier failed and another absent is failed", ( const graph = sampleGraph([ { id: "spec:sample.evaluate", - opening: "Sc L0-1 · domain tier.", + opening: "Sc L0-1 · pure test tier.", files: [ "tests/pure/evaluate.test.ts", "tests/pure/evaluate-again.test.ts", @@ -515,17 +515,17 @@ test("pure: a graph that throws while it is read leaves the check unable to answ const evaluate: SampleExample = { id: "spec:sample.evaluate", - opening: "Sc L0-1 · domain tier.", + opening: "Sc L0-1 · pure test tier.", files: ["tests/pure/evaluate.test.ts"], }; const transition: SampleExample = { id: "spec:sample.transition", - opening: "Sc L1-1 · native tier · fixture composition.", + opening: "Sc L1-1 · native backend tier · fixture composition.", files: ["tests/native/transition.test.ts"], }; const production: SampleExample = { id: "spec:sample.production", - opening: "Sc L2-9 · end to end tier · production composition.", + opening: "Sc L2-9 · native backend tier · production composition.", files: ["tests/native/production.test.ts"], }; const cannotAnswer = (graph: ScenarioGraph) => @@ -542,7 +542,7 @@ test("pure: a tier part with anything after the word tier is unreadable, and the evaluate, { ...transition, - opening: "Sc L1-1 · native tierBROKEN · fixture composition.", + opening: "Sc L1-1 · native backend tierBROKEN · fixture composition.", }, production, ]), @@ -550,6 +550,19 @@ test("pure: a tier part with anything after the word tier is unreadable, and the expect(line).toContain("spec:sample.transition names no tier"); }); +test("pure: an opening line that names a tier by an old name names no tier, and the check exits 2", async () => { + for (const tier of ["domain", "simulator", "native", "build", "end to end"]) { + const line = await cannotAnswer( + sampleGraph([ + evaluate, + { ...transition, opening: `Sc L1-1 · ${tier} tier.` }, + production, + ]), + ); + expect(line).toContain("spec:sample.transition names no tier"); + } +}); + test("pure: a malformed sibling of a required row's example makes the check exit 2, and never drops out", async () => { const line = await cannotAnswer( sampleGraph([ @@ -557,7 +570,7 @@ test("pure: a malformed sibling of a required row's example makes the check exit transition, { id: "spec:sample.transition-sibling", - opening: "Sc L1-1: native tier · fixture composition.", + opening: "Sc L1-1: native backend tier · fixture composition.", files: ["tests/native/transition.test.ts"], }, production, @@ -568,9 +581,9 @@ test("pure: a malformed sibling of a required row's example makes the check exit test("pure: every opening line that begins with Sc reads whole, of a required row or not, composition included", async () => { for (const opening of [ - "Sc L3-1 · native tier · fixture compositionBROKEN.", - "Sc L3-1 · natve tier.", - "Sc L3-1x · native tier.", + "Sc L3-1 · native backend tier · fixture compositionBROKEN.", + "Sc L3-1 · natve backend tier.", + "Sc L3-1x · native backend tier.", ]) await cannotAnswer( sampleGraph([ @@ -584,16 +597,16 @@ test("pure: every opening line that begins with Sc reads whole, of a required ro // not begin with Sc is no scenario's. const answer = await answered( sampleGraph([ - { ...evaluate, opening: "Sc L0-1 · domain tier · the only case." }, + { ...evaluate, opening: "Sc L0-1 · pure test tier · the only case." }, { ...transition, opening: - "Sc L1-1 · native tier · fixture composition · first of two cases.", + "Sc L1-1 · native backend tier · fixture composition · first of two cases.", }, production, { id: "spec:sample.probe", - opening: "Probe 1 · native tierX.", + opening: "Probe 1 · native backend tierX.", files: [], }, ]), @@ -673,7 +686,8 @@ test("pure: the newest record is chosen by the time of day as well as the date", test("pure: every example of a required row is judged, not only the first", () => { const second: SampleExample = { id: "spec:sample.transition-second", - opening: "Sc L1-1 · native tier · fixture composition · the second case.", + opening: + "Sc L1-1 · native backend tier · fixture composition · the second case.", files: ["tests/native/transition-second.test.ts"], }; const record = sampleRecord(); @@ -748,17 +762,17 @@ test("pure: the second verifier of an example is inspected as well as the first" ).toBe("absent"); }); -test("pure: a build scenario passes only on a result of the types project, and a simulator scenario only on one of the simulator project", () => { +test("pure: a compiled scenario passes only on a result of the types project, and a convex-test scenario only on one of the simulator project", () => { const graph = sampleGraph( [ { id: "spec:sample.build", - opening: "Sc L2-4 · build tier.", + opening: "Sc L2-4 · compiled tier.", files: ["tests/types/broken.test-d.ts"], }, { id: "spec:sample.simulated", - opening: "Sc L1-9 · simulator tier.", + opening: "Sc L1-9 · convex-test tier.", files: ["tests/simulator/simulated.test.ts"], }, ], @@ -792,7 +806,7 @@ test("pure: a build scenario passes only on a result of the types project, and a expect(statuses("native", "pure")).toEqual(["failed", "failed"]); }); -test("pure: acceptanceRows names the rows the doc's table gives Layer 0, 1 or 2, and every native example of them that has a verifier names its composition", async () => { +test("pure: acceptanceRows names the rows the doc's table gives Layer 0, 1 or 2, and every native backend example of them that has a verifier names its composition", async () => { const doc = await readFile( join(root, "docs/convex-transactional-domain-platform-decisions.md"), "utf8", @@ -818,7 +832,7 @@ test("pure: acceptanceRows names the rows the doc's table gives Layer 0, 1 or 2, scenarios .filter( (scenario) => - (scenario.tier === "native" || scenario.tier === "end to end") && + scenario.tier === "native backend" && scenario.verifiers.length > 0 && scenario.composition === null, ) diff --git a/tests/pure/command.test.ts b/tests/pure/command.test.ts index 988f953..17d7acb 100644 --- a/tests/pure/command.test.ts +++ b/tests/pure/command.test.ts @@ -184,7 +184,7 @@ test("pure: reject and refuseTransient throw ConvexErrors whose data fits errorD const rejection = thrown(() => reject({ code: "forbidden", - commandType: "CreateDocument", + entry: "CreateDocument", message: "No", details: { reason: "no_grant" }, }), @@ -193,7 +193,7 @@ test("pure: reject and refuseTransient throw ConvexErrors whose data fits errorD expect((rejection as ConvexError).data).toEqual({ kind: "rejection", code: "forbidden", - commandType: "CreateDocument", + entry: "CreateDocument", message: "No", details: { reason: "no_grant" }, }); @@ -218,7 +218,7 @@ test("pure: reject throws a plain Error when its details measure above 16,384 by expect(getConvexSize(atBound)).toBe(16384); const data = { code: "forbidden" as const, - commandType: "CreateDocument", + entry: "CreateDocument", message: "No", }; expect(thrown(() => reject({ ...data, details: atBound }))).toBeInstanceOf( @@ -232,7 +232,7 @@ test("pure: reject throws a plain Error when its details measure above 16,384 by expect(String(above)).toContain("16385 bytes of details"); }); -test("pure: normalizeThrown wraps a bare kernel rejection with the discriminator and the command type", () => { +test("pure: normalizeThrown wraps a bare kernel rejection with the discriminator and the entry", () => { const bare = new ConvexError({ code: "invalidTransition", message: "A document cannot ship from draft", @@ -244,14 +244,14 @@ test("pure: normalizeThrown wraps a bare kernel rejection with the discriminator expect(rethrown).toBeInstanceOf(ConvexError); expect((rethrown as ConvexError).data).toEqual({ kind: "rejection", - commandType: "ShipDocument", + entry: "ShipDocument", code: "invalidTransition", message: "A document cannot ship from draft", details: { from: "draft", trigger: "ship" }, }); }); -test("pure: normalizeThrown rejects another command name and rethrows a plain error, a transient refusal, a ConvexError carrying other data and a thrown string unchanged", () => { +test("pure: normalizeThrown rejects another entry's name and rethrows a plain error, a transient refusal, a ConvexError carrying other data and a thrown string unchanged", () => { const passed = [ new Error("Fault injected"), new ConvexError({ kind: "transient", code: "capacity", message: "Full" }), @@ -262,7 +262,7 @@ test("pure: normalizeThrown rejects another command name and rethrows a plain er const mismatched = new ConvexError({ kind: "rejection", code: "forbidden", - commandType: "Other", + entry: "Other", message: "No", }); const normalized = thrown(() => @@ -287,14 +287,14 @@ test("pure: normalizeThrown wraps a kernel rejection with a platform code that t const rethrown = thrown(() => normalizeThrown(bare, "ShipDocument", [])); expect((rethrown as ConvexError).data).toEqual({ kind: "rejection", - commandType: "ShipDocument", + entry: "ShipDocument", code: "staleVersion", message: "Stream document/doc-1 is at version 2, not 1", details: { expected: 1, current: 2 }, }); }); -test("pure: normalizeThrown rethrows a kernel rejection whose code is neither a platform code nor in rejections as a plain Error naming the code and the command type", () => { +test("pure: normalizeThrown rethrows a kernel rejection whose code is neither a platform code nor in rejections as a plain Error naming the code and the entry", () => { const bare = new ConvexError({ code: "insufficientStock", message: "Cannot claim 2 when 1 are on hand", @@ -311,7 +311,7 @@ test("pure: normalizeThrown rethrows a kernel rejection whose code is neither a } }); -test("pure: normalizeThrown rethrows a kernel rejection whose details measure above 16,384 bytes as a plain Error naming the code and the command type", () => { +test("pure: normalizeThrown rethrows a kernel rejection whose details measure above 16,384 bytes as a plain Error naming the code and the entry", () => { const bare = (documentId: string) => new ConvexError({ code: "invalidInput", @@ -337,14 +337,14 @@ test("pure: normalizeThrown rethrows a kernel rejection whose details measure ab test("pure: classifyThrown tells a rejection from a transient refusal from a technical failure", () => { const rejection = thrown(() => - reject({ code: "forbidden", commandType: "C", message: "No" }), + reject({ code: "forbidden", entry: "C", message: "No" }), ); expect(classifyThrown(rejection)).toEqual({ kind: "rejection", data: { kind: "rejection", code: "forbidden", - commandType: "C", + entry: "C", message: "No", }, }); @@ -363,7 +363,7 @@ test("pure: classifyThrown tells a rejection from a transient refusal from a tec new ConvexError({ kind: "rejection", code: "forbidden", - commandType: "C", + entry: "C", message: "No", details: ["listed"], }), diff --git a/tests/pure/rejection-validation.test.ts b/tests/pure/rejection-validation.test.ts index 133c0df..4dd6ef1 100644 --- a/tests/pure/rejection-validation.test.ts +++ b/tests/pure/rejection-validation.test.ts @@ -2,15 +2,15 @@ import { ConvexError, getConvexSize, type Value } from "convex/values"; import { expect, test } from "vitest"; import { normalizeThrown, classifyThrown } from "../../src/command/index.js"; -const commandType = "CreateDocument"; +const entry = "CreateDocument"; const message = "Refused"; const bare = (code: string, extra: Record = {}) => new ConvexError({ code, message, ...extra }); const wire = ( code = "forbidden", - name = commandType, + name = entry, extra: Record = {}, -) => bare(code, { kind: "rejection", commandType: name, ...extra }); +) => bare(code, { kind: "rejection", entry: name, ...extra }); function details(bytes: number, multibyte = false) { const overhead = getConvexSize({ text: "" }); const count = bytes - overhead; @@ -24,7 +24,7 @@ function details(bytes: number, multibyte = false) { } function normalized(error: unknown) { try { - normalizeThrown(error, commandType, ["invalidTransition"]); + normalizeThrown(error, entry, ["invalidTransition"]); } catch (thrown) { return thrown; } @@ -51,7 +51,7 @@ const cases: Case[] = [ name: "bare undeclared code", error: bare("notDeclared"), result: "technical", - contains: ["notDeclared", commandType], + contains: ["notDeclared", entry], }, { name: "wire platform code", error: wire(), result: "same" }, { @@ -63,47 +63,47 @@ const cases: Case[] = [ name: "wire undeclared code", error: wire("notDeclared"), result: "technical", - contains: ["notDeclared", commandType], + contains: ["notDeclared", entry], }, { - name: "wire wrong command", + name: "wire wrong entry", error: wire("forbidden", "SomeOtherCommand"), result: "technical", - contains: ["SomeOtherCommand", commandType], + contains: ["SomeOtherCommand", entry], }, { - name: "wire command that differs only in case", + name: "wire entry that differs only in case", error: wire("forbidden", "createDocument"), result: "technical", - contains: ["createDocument", commandType], + contains: ["createDocument", entry], }, { - name: "wire code checked before command", + name: "wire code checked before entry", error: wire("notDeclared", "SomeOtherCommand"), result: "technical", - contains: ["notDeclared", commandType], + contains: ["notDeclared", entry], }, { name: "wire details at 16384 bytes", - error: wire("forbidden", commandType, { details: details(16384) }), + error: wire("forbidden", entry, { details: details(16384) }), result: "same", }, { name: "wire details at 16385 bytes", - error: wire("forbidden", commandType, { details: details(16385) }), + error: wire("forbidden", entry, { details: details(16385) }), result: "technical", - contains: ["16385", commandType], + contains: ["16385", entry], }, { name: "wire multibyte details at 16384 bytes", - error: wire("forbidden", commandType, { details: details(16384, true) }), + error: wire("forbidden", entry, { details: details(16384, true) }), result: "same", }, { name: "wire multibyte details at 16386 bytes", - error: wire("forbidden", commandType, { details: details(16386, true) }), + error: wire("forbidden", entry, { details: details(16386, true) }), result: "technical", - contains: ["16386", commandType], + contains: ["16386", entry], }, { name: "bare details at 16384 bytes", @@ -114,25 +114,25 @@ const cases: Case[] = [ name: "bare details at 16385 bytes", error: bare("invalidTransition", { details: details(16385) }), result: "technical", - contains: ["16385", commandType], + contains: ["16385", entry], }, { - name: "wire missing command", + name: "wire missing entry", error: bare("forbidden", { kind: "rejection" }), result: "technical", - contains: [commandType], + contains: [entry], }, { name: "wire numeric code", - error: wire("forbidden", commandType, { code: 1 }), + error: wire("forbidden", entry, { code: 1 }), result: "technical", - contains: [commandType], + contains: [entry], }, { name: "wire extra field", - error: wire("forbidden", commandType, { extra: 1 }), + error: wire("forbidden", entry, { extra: 1 }), result: "technical", - contains: [commandType], + contains: [entry], }, { name: "transient refusal", @@ -150,37 +150,37 @@ const cases: Case[] = [ name: "wire inherited Object code", error: wire("toString"), result: "technical", - contains: ["toString", commandType], + contains: ["toString", entry], }, { name: "wire details that are not a record", - error: wire("forbidden", commandType, { details: [] }), + error: wire("forbidden", entry, { details: [] }), result: "technical", - contains: [commandType], + contains: [entry], }, { name: "bare details that are not a record", error: bare("invalidTransition", { details: ["listed"] }), result: "technical", - contains: [commandType], + contains: [entry], }, { name: "bare missing message", error: new ConvexError({ code: "invalidTransition" }), result: "technical", - contains: [commandType], + contains: [entry], }, { name: "bare numeric code", error: bare("invalidTransition", { code: 1 }), result: "technical", - contains: [commandType], + contains: [entry], }, { name: "bare extra field", error: bare("invalidTransition", { extra: 1 }), result: "technical", - contains: [commandType], + contains: [entry], }, { name: "ConvexError carrying a list that has a code", @@ -195,10 +195,10 @@ const cases: Case[] = [ result: "same", }, { - name: "wire command checked before details", + name: "wire entry checked before details", error: wire("forbidden", "SomeOtherCommand", { details: details(16385) }), result: "technical", - contains: ["SomeOtherCommand", commandType], + contains: ["SomeOtherCommand", entry], }, ]; // spec:command.outcome-boundary, fnNormalizeThrown. @@ -209,7 +209,7 @@ test.each(cases)("pure: normalizes $name", ({ error, result, contains }) => { expect(thrown).toBeInstanceOf(ConvexError); expect((thrown as ConvexError).data).toEqual({ kind: "rejection", - commandType, + entry, ...(error as ConvexError>).data, }); } else { diff --git a/tests/pure/stock-item-id-bound.test.ts b/tests/pure/stock-item-id-bound.test.ts index 7512d77..1e20452 100644 --- a/tests/pure/stock-item-id-bound.test.ts +++ b/tests/pure/stock-item-id-bound.test.ts @@ -140,12 +140,12 @@ test("pure: the pipeline refuses a long ID before any read and checks the line c ...refusal(0, 65), kind: "rejection", code: "invalidInput", - commandType: "PlaceOrder", + entry: "PlaceOrder", } : { kind: "rejection", code: "operationTooLarge", - commandType: "PlaceOrder", + entry: "PlaceOrder", message: "PlaceOrder takes at most 100 items, not 101", details: { items: 101, maxItems: 100 }, }, diff --git a/tests/simulator/command.test.ts b/tests/simulator/command.test.ts index 6893246..25bec71 100644 --- a/tests/simulator/command.test.ts +++ b/tests/simulator/command.test.ts @@ -202,7 +202,7 @@ describe("the public entry", () => { ).toEqual({ kind: "rejection", code: "unauthenticated", - commandType: "CreateDocument", + entry: "CreateDocument", message: "CreateDocument needs an authenticated caller", }); const stranger = t.withIdentity({ issuer, subject: "user-2" }); @@ -213,7 +213,7 @@ describe("the public entry", () => { ).toMatchObject({ kind: "rejection", code: "forbidden", - commandType: "CreateDocument", + entry: "CreateDocument", details: { reason: "no_grant" }, }); const narrow = await caller(t, "user-3", { @@ -288,7 +288,7 @@ describe("the public entry", () => { ).toMatchObject({ kind: "rejection", code: "invalidInput", - commandType: "CreateDocument", + entry: "CreateDocument", details: { field: "requestKey", length: 257, limit: 256 }, }); expect( @@ -327,7 +327,7 @@ describe("the public entry", () => { ), ).toMatchObject({ code: "operationTooLarge", - commandType: "AddStock", + entry: "AddStock", details: { items: 101, maxItems: 100 }, }); expect( @@ -341,7 +341,7 @@ describe("the public entry", () => { test( name( - "a context's rejection reaches the caller with the discriminator and the command type, and commits nothing", + "a context's rejection reaches the caller with the discriminator and the entry, and commits nothing", ), async () => { const t = app(); @@ -357,7 +357,7 @@ describe("the public entry", () => { ), ).toEqual({ kind: "rejection", - commandType: "ShipDocument", + entry: "ShipDocument", code: "invalidTransition", message: "A document cannot ship from draft", details: { from: "draft", trigger: "ship" }, @@ -387,7 +387,7 @@ describe("the public entry", () => { ), ).toMatchObject({ kind: "rejection", - commandType: "CreateDocument", + entry: "CreateDocument", code: "entityExists", details: { existing: "doc-1", current: 1 }, }); @@ -399,7 +399,7 @@ describe("the public entry", () => { input: { documentId: "doc-1", expectedVersion: 0 }, }), ), - ).toMatchObject({ code: "entityExists", commandType: "FailIfDecided" }); + ).toMatchObject({ code: "entityExists", entry: "FailIfDecided" }); expect( await errorData( user.mutation(api.depotCommands.failIfDecided, { @@ -687,7 +687,7 @@ describe("receipts", () => { ).toEqual({ kind: "rejection", code: "idempotencyConflict", - commandType: "CreateDocument", + entry: "CreateDocument", message: "This request key was used with other input", }); expect(await receipts(t)).toEqual(before); @@ -719,7 +719,7 @@ describe("receipts", () => { ), ).toEqual({ kind: "rejection", - commandType: "CreateDocument", + entry: "CreateDocument", code: "unsupportedContractVersion", message: "This request key was used under another contract version", }); @@ -808,7 +808,7 @@ describe("receipts", () => { expect(data).toEqual({ kind: "rejection", code: "forbidden", - commandType: "CreateDocument", + entry: "CreateDocument", message: "The caller may not run CreateDocument in this tenant", details: { reason: "no_grant" }, }); @@ -1131,7 +1131,7 @@ describe("the internal entry", () => { input: { documentId: "doc-2", title: "Report" }, }), ), - ).toMatchObject({ code: "forbidden", commandType: "CreateDocument" }); + ).toMatchObject({ code: "forbidden", entry: "CreateDocument" }); }, ); diff --git a/tests/simulator/context-queries.test.ts b/tests/simulator/context-queries.test.ts index 0fdcee3..999331a 100644 --- a/tests/simulator/context-queries.test.ts +++ b/tests/simulator/context-queries.test.ts @@ -427,7 +427,7 @@ describe("the parent relays a context query across the component boundary", () = expect(refused).toBeInstanceOf(ConvexError); expect((refused as ConvexError).data).toMatchObject({ code: "forbidden", - commandType: "getDocument", + entry: "getDocument", details: { reason: "subject_mismatch" }, }); expect(await get("tenant-wide", "doc-2")).toMatchObject({ @@ -470,7 +470,7 @@ describe("the parent relays a context query across the component boundary", () = expect((error as ConvexError).data).toMatchObject({ kind: "rejection", code, - commandType: relay, + entry: relay, }); } } diff --git a/tests/simulator/envelope-bounds.test.ts b/tests/simulator/envelope-bounds.test.ts index 8a4fdda..6ccbde6 100644 --- a/tests/simulator/envelope-bounds.test.ts +++ b/tests/simulator/envelope-bounds.test.ts @@ -52,7 +52,7 @@ test.each([ expect(data).toMatchObject({ kind: "rejection", code: "invalidInput", - commandType: "CreateDocument", + entry: "CreateDocument", }); expect(data.details).toEqual({ field: "correlationId", @@ -224,7 +224,7 @@ test.each(fields)( ); expect(data).toMatchObject({ code: "invalidInput", - commandType: "CreateDocument", + entry: "CreateDocument", }); expect(data.details).toEqual({ field, length: limit + 1, limit }); await expectEmpty(t); @@ -242,7 +242,7 @@ test.each(fields)( ); expect(data).toMatchObject({ code: "invalidInput", - commandType: "CreateDocument", + entry: "CreateDocument", }); expect(data.details).toEqual({ field, length: limit + 1, limit }); await expectEmpty(t); @@ -409,7 +409,7 @@ test.each([ expect(data).toMatchObject({ kind: "rejection", code: "invalidInput", - commandType: "CreateDocument", + entry: "CreateDocument", }); expect(data.details).toEqual({ field: "streamId", length, limit: 256 }); await expectEmpty(t); diff --git a/tests/simulator/example.test.ts b/tests/simulator/example.test.ts index 7d41a2b..4e73857 100644 --- a/tests/simulator/example.test.ts +++ b/tests/simulator/example.test.ts @@ -216,7 +216,7 @@ test( expect(await errorData(send())).toEqual({ kind: "rejection", code: "insufficientStock", - commandType: "PlaceOrder", + entry: "PlaceOrder", message: "Cannot allocate 3 when 2 are available", details: { requested: 3, available: 2 }, }); @@ -245,7 +245,7 @@ test( input: { orderId: "order-1", lines: [line("sku-1", 3)] }, }), ), - ).toMatchObject({ code: "entityExists", commandType: "PlaceOrder" }); + ).toMatchObject({ code: "entityExists", entry: "PlaceOrder" }); }, ); @@ -280,7 +280,7 @@ test( ).toBeNull(); expect(await errorData(send([]))).toMatchObject({ code: "invalidInput", - commandType: "PlaceOrder", + entry: "PlaceOrder", message: "An order needs at least one line", }); // rebuild.sdp.md:115,133: installing returns a generation ID; its number is on the row. @@ -302,7 +302,7 @@ test( expect(await errorData(t.query(api.orderQueries.getOrder, args))).toEqual({ kind: "rejection", code: "unauthenticated", - commandType: "getOrder", + entry: "getOrder", message: "getOrder needs an authenticated caller", }); const stranger = t.withIdentity({ issuer, subject: "user-2" }); @@ -316,7 +316,7 @@ test( ).toEqual({ kind: "rejection", code: "forbidden", - commandType: "listOrders", + entry: "listOrders", message: "The caller may not read listOrders in this tenant", details: { reason: "no_grant" }, }); @@ -324,7 +324,7 @@ test( const reader = await caller(t, "user-3", [readOrdersPermission], "t-2"); expect( await errorData(reader.query(api.orderQueries.getOrder, args)), - ).toMatchObject({ code: "forbidden", commandType: "getOrder" }); + ).toMatchObject({ code: "forbidden", entry: "getOrder" }); const summaryArgs = { tenantId: "t-1", status: "placed" as const, @@ -335,7 +335,7 @@ test( await errorData( stranger.query(api.readModels.listOrderSummaries, summaryArgs), ), - ).toMatchObject({ code: "forbidden", commandType: "listOrderSummaries" }); + ).toMatchObject({ code: "forbidden", entry: "listOrderSummaries" }); const granted = await caller(t, "user-3", [readOrdersPermission]); expect(await granted.query(api.orderQueries.getOrder, args)).toBeNull(); const list = failure( @@ -370,7 +370,7 @@ test( ).toEqual({ kind: "rejection", code: "invalidQuantity", - commandType: "ReceiveStock", + entry: "ReceiveStock", message: "A quantity must be a positive whole number, not 0", details: { quantity: 0 }, }); @@ -424,7 +424,7 @@ test( ), ).toMatchObject({ code: "forbidden", - commandType: "getOrder", + entry: "getOrder", details: { reason: "subject_mismatch" }, }); }, @@ -620,7 +620,7 @@ test( ).toEqual({ kind: "rejection", code: "orderAlreadyCancelled", - commandType: "CancelOrder", + entry: "CancelOrder", message: "The order is already cancelled", }); expect(await after()).toEqual(before); @@ -642,7 +642,7 @@ test( ).toEqual({ kind: "rejection", code: "orderNotFound", - commandType: "CancelOrder", + entry: "CancelOrder", message: "The order does not exist", }); expect( @@ -685,7 +685,7 @@ test( ).toEqual({ kind: "rejection", code: "insufficientAllocation", - commandType: "CancelOrder", + entry: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 }, }); @@ -722,7 +722,7 @@ test( ).toEqual({ kind: "rejection", code: "forbidden", - commandType: "CancelOrder", + entry: "CancelOrder", message: "The caller may not run CancelOrder in this tenant", details: { reason: "no_grant" }, }); @@ -731,7 +731,7 @@ test( await errorData( other.mutation(api.ordering.cancelOrder, cancelCall("k-2", "order-2")), ), - ).toMatchObject({ code: "forbidden", commandType: "CancelOrder" }); + ).toMatchObject({ code: "forbidden", entry: "CancelOrder" }); expect(await stockItem(t)).toMatchObject({ allocated: 3 }); expect(await summaries(t)).toMatchObject([{ status: "placed" }]); expect( diff --git a/tests/simulator/read-model.test.ts b/tests/simulator/read-model.test.ts index 53eaa72..6eaa18f 100644 --- a/tests/simulator/read-model.test.ts +++ b/tests/simulator/read-model.test.ts @@ -682,7 +682,7 @@ describe("step 1", () => { expect(await errorData(create(tenantId, requestKey))).toEqual({ kind: "rejection", code: "invalidInput", - commandType: "CreateDocument", + entry: "CreateDocument", message: `${field} has at most 256 bytes of UTF-8`, details: { field, length: 257, limit: 256 }, }); @@ -728,14 +728,14 @@ describe("authorizeQuery and the parent list", () => { expect(await errorData(list(t))).toEqual({ kind: "rejection", code: "unauthenticated", - commandType: "listDocumentSummaries", + entry: "listDocumentSummaries", message: "listDocumentSummaries needs an authenticated caller", }); const stranger = await caller(t, "user-2", permissions.documents); expect(await errorData(list(stranger))).toEqual({ kind: "rejection", code: "forbidden", - commandType: "listDocumentSummaries", + entry: "listDocumentSummaries", message: "The caller may not read listDocumentSummaries in this tenant", details: { reason: "no_grant" }, }); diff --git a/tests/simulator/receipt-disclosure.test.ts b/tests/simulator/receipt-disclosure.test.ts index e9bd051..58c5545 100644 --- a/tests/simulator/receipt-disclosure.test.ts +++ b/tests/simulator/receipt-disclosure.test.ts @@ -31,7 +31,7 @@ test( ); expect.soft(data).toEqual({ kind: "rejection", - commandType: "CreateDocument", + entry: "CreateDocument", code: "idempotencyConflict", message: "This request key was used with other input", }); @@ -53,7 +53,7 @@ test( ).toMatchObject({ kind: "rejection", code: "forbidden", - commandType: "CreateDocument", + entry: "CreateDocument", }); expect(await stored(t)).toEqual(before); }, @@ -95,7 +95,7 @@ test.each(["same input", "other subject"])( ); expect.soft(data).toEqual({ kind: "rejection", - commandType: "CreateDocument", + entry: "CreateDocument", code: "unsupportedContractVersion", message: "This request key was used under another contract version", }); diff --git a/tests/types/reject-platform-codes.test-d.ts b/tests/types/reject-platform-codes.test-d.ts index 23cdaed..2de92e8 100644 --- a/tests/types/reject-platform-codes.test-d.ts +++ b/tests/types/reject-platform-codes.test-d.ts @@ -5,10 +5,10 @@ test("reject accepts only platform rejection codes", () => { expectTypeOf< Parameters[0]["code"] >().toEqualTypeOf(); - reject({ code: "forbidden", commandType: "X", message: "m" }); + reject({ code: "forbidden", entry: "X", message: "m" }); // @ts-expect-error A domain code must go through the running command's declaration. - reject({ code: "notDeclared", commandType: "X", message: "m" }); + reject({ code: "notDeclared", entry: "X", message: "m" }); const code: string = "forbidden"; // @ts-expect-error An arbitrary string is not a platform code. - reject({ code, commandType: "X", message: "m" }); + reject({ code, entry: "X", message: "m" }); }); From cd7758eac83c3dfbb340b3f6b174cd2b378ee9b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:40:25 +0200 Subject: [PATCH 117/155] build: hosted the backup archive names The harness's members that name the exported file take the language's word: exportBackupArchive and importBackupArchive, in harness/backup-archive.ts, with the CLI helper and the temporary deploy renamed to what they do. Snapshot stays only for a transaction's view. Callers, tests and the Spec's AdminAccess follow. Co-Authored-By: Claude Opus 5.5 --- design/specs/platform/native-harness.sdp.md | 2 +- harness/admin.ts | 17 +++---- harness/{snapshot.ts => backup-archive.ts} | 28 ++++++------ ...t-archive.ts => backup-archive-entries.ts} | 0 ....ts => backup-archive-replacement.test.ts} | 44 +++++++++---------- ...heduler.ts => backup-archive-scheduler.ts} | 2 +- tests/native/scheduler-cases.ts | 6 +-- tests/native/scheduler-restore.ts | 4 +- tests/pure/child.test.ts | 8 ++-- tests/pure/harness-rules.test.ts | 16 +++---- 10 files changed, 62 insertions(+), 65 deletions(-) rename harness/{snapshot.ts => backup-archive.ts} (74%) rename tests/native/{snapshot-archive.ts => backup-archive-entries.ts} (100%) rename tests/native/{snapshot-replacement.test.ts => backup-archive-replacement.test.ts} (74%) rename tests/native/{snapshot-scheduler.ts => backup-archive-scheduler.ts} (96%) diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 6432039..e9228dc 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -79,7 +79,7 @@ The harness is plain test support with no Convex function in it. The entries who - adminReads: [extension] the harness reads a table of the app or of a component, and the function log's completion records, with the admin key; a table read returns every document with every Convex value intact or fails, never a sample, and is several queries, not one snapshot; a log read returns every completion record after a point the test marks or fails, and a mark taken before a restart cannot be read after it; a completion record carries at least `identifier`, `componentPath`, `requestId`, `executionTime` in seconds, `cachedResult`, `error` and `logLines` as an array of strings (E-15) - typeFixtureIssuer: [extension] `interface FixtureIssuer { issuer: string; applicationID: string; jwks: string; token(subject: string): Promise }` (E-13) - typeBackend: [extension] `interface Backend { url: string; adminKey: string; issuer: FixtureIssuer; admin: AdminAccess; stop(): Promise; kill(): Promise; restart(): Promise; dispose(): Promise; facts(): BackendFacts }`; `stop` and `kill` keep the storage; `dispose` alone removes it and is final, so a disposed backend cannot be started again; `adminKey` is there for the one example that shows why no caller carries it (E-15) -- typeAdminAccess: [extension] `interface AdminAccess { deploy(): Promise; deployTemporary(directory: string, signal?: AbortSignal): Promise<{ command: string[]; stdout: string; stderr: string }>; codegen(): Promise; setEnvironment(variables: Record): Promise; environment(): Promise>; run(name: string, args?: Record, options?: { component?: string }): Promise; readTable(table: string, options?: { component?: string; pageSize?: number }): Promise[]>; writeTable(table: string, write: { insert: Record } | { patch: string; fields: Record } | { delete: string }, options?: { component?: string }): Promise; logMark(): Promise; completionsSince(mark: LogMark, until: (records: readonly CompletionRecord[]) => boolean, timeoutMs?: number): Promise; exportSnapshot(path: string): Promise; replaceSnapshot(path: string): Promise }` holds every use of the admin key and takes no identity; the two snapshot members run the Convex CLI's export and import with replacement against this backend, and a test uses them and never the admin key itself; `deployTemporary` deploys a temporary copy, stops its deploy child when the caller's signal or the backend's own aborts, returns its redacted command and clears the plain composition claim (E-15) +- typeAdminAccess: [extension] `interface AdminAccess { deploy(): Promise; deployTemporary(directory: string, signal?: AbortSignal): Promise<{ command: string[]; stdout: string; stderr: string }>; codegen(): Promise; setEnvironment(variables: Record): Promise; environment(): Promise>; run(name: string, args?: Record, options?: { component?: string }): Promise; readTable(table: string, options?: { component?: string; pageSize?: number }): Promise[]>; writeTable(table: string, write: { insert: Record } | { patch: string; fields: Record } | { delete: string }, options?: { component?: string }): Promise; logMark(): Promise; completionsSince(mark: LogMark, until: (records: readonly CompletionRecord[]) => boolean, timeoutMs?: number): Promise; exportBackupArchive(path: string): Promise; importBackupArchive(path: string): Promise }` holds every use of the admin key and takes no identity; the two backup archive members run the Convex CLI's export of a backup archive and its import with replacement against this backend, and a test uses them and never the admin key itself; `deployTemporary` deploys a temporary copy, stops its deploy child when the caller's signal or the backend's own aborts, returns its redacted command and clears the plain composition claim (E-15) - fnOrdinaryClient: [extension] `ordinaryClient(url: string, options?: { token?: string; fetch?: typeof fetch }): ConvexHttpClient` (E-13) - fnOrdinarySocketClient: [extension] `ordinarySocketClient(url: string, options?: { token?: string; webSocket?: typeof WebSocket }): ConvexClient` (E-13) - fnCountingFetch: [extension] `countingFetch(options: { loseResponses: boolean }): { fetch: typeof fetch; requests(): number }` delivers each request and, when it loses responses, throws after the backend has answered (E-15) diff --git a/harness/admin.ts b/harness/admin.ts index 5bcceb7..22abed5 100644 --- a/harness/admin.ts +++ b/harness/admin.ts @@ -10,8 +10,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { redact, runChild } from "./child.js"; -import { snapshotCommand, createSnapshotAccess } from "./snapshot.js"; -import type { SnapshotCommand } from "./snapshot.js"; +import { isolatedCli, createBackupArchiveAccess } from "./backup-archive.js"; +import type { CliOutput } from "./backup-archive.js"; import { projectDirectory } from "./composition.js"; import type { Composition } from "./composition.js"; const anchor = codeAnchor({ @@ -57,10 +57,7 @@ export interface CompletionRecord { export interface AdminAccess { deploy(): Promise; // The child stops when the caller's signal or the backend's own aborts. - deployTemporary( - directory: string, - signal?: AbortSignal, - ): Promise; + deployTemporary(directory: string, signal?: AbortSignal): Promise; codegen(): Promise; setEnvironment(variables: Readonly>): Promise; environment(): Promise>; @@ -82,8 +79,8 @@ export interface AdminAccess { | { delete: string }, options?: { component?: string }, ): Promise; - exportSnapshot(path: string): Promise; - replaceSnapshot(path: string): Promise; + exportBackupArchive(path: string): Promise; + importBackupArchive(path: string): Promise; logMark(): Promise; completionsSince( mark: LogMark, @@ -214,7 +211,7 @@ export function createAdminAccess( }); } return { - ...createSnapshotAccess(target), + ...createBackupArchiveAccess(target), async deploy() { await cli("deploy"); state.deployed = target.composition; @@ -223,7 +220,7 @@ export function createAdminAccess( const signals = [signal, target.signal].filter( (one): one is AbortSignal => one !== undefined, ); - const result = await snapshotCommand( + const result = await isolatedCli( target, [...commandArgs.deploy], signals.length === 0 ? undefined : AbortSignal.any(signals), diff --git a/harness/snapshot.ts b/harness/backup-archive.ts similarity index 74% rename from harness/snapshot.ts rename to harness/backup-archive.ts index d404e71..9046904 100644 --- a/harness/snapshot.ts +++ b/harness/backup-archive.ts @@ -10,26 +10,26 @@ import type { AdminAccess, AdminTarget } from "./admin.js"; import type { Backend } from "./backend.js"; import { redact, runChild } from "./child.js"; const anchor = codeAnchor({ - id: codeAnchorId("impl:platform.native-harness.snapshot"), - label: "snapshot export and replacement import through the Convex CLI", + id: codeAnchorId("impl:platform.native-harness.backup-archive"), + label: "backup archive export and replacement import through the Convex CLI", satisfies: ref("spec:platform.native-harness"), }); void anchor; -export interface SnapshotCommand { +export interface CliOutput { command: string[]; stdout: string; stderr: string; } // Uses the installed CLI and an isolated home, with no inherited deployment selection. -export async function snapshotCommand( +export async function isolatedCli( backend: Pick, args: string[], signal: AbortSignal | undefined, directory = join(import.meta.dirname, ".."), -): Promise { - const home = await mkdtemp(join(tmpdir(), "convex-snapshot-")); +): Promise { + const home = await mkdtemp(join(tmpdir(), "convex-cli-")); const selection = ["--url", backend.url, "--admin-key", backend.adminKey]; const file = process.execPath; const command = [ @@ -53,24 +53,24 @@ export async function snapshotCommand( } } -export function deploySnapshotFixture( +export function deployTemporaryCopy( backend: Pick, directory: string, signal: AbortSignal, -): Promise { +): Promise { signal.throwIfAborted(); return backend.admin.deployTemporary(directory, signal); } -export function createSnapshotAccess( +export function createBackupArchiveAccess( target: Pick, -): Pick { +): Pick { return { - async exportSnapshot(path) { - await snapshotCommand(target, ["export", "--path", path], target.signal); + async exportBackupArchive(path) { + await isolatedCli(target, ["export", "--path", path], target.signal); }, - async replaceSnapshot(path) { - await snapshotCommand( + async importBackupArchive(path) { + await isolatedCli( target, ["import", "--replace", "--yes", path], target.signal, diff --git a/tests/native/snapshot-archive.ts b/tests/native/backup-archive-entries.ts similarity index 100% rename from tests/native/snapshot-archive.ts rename to tests/native/backup-archive-entries.ts diff --git a/tests/native/snapshot-replacement.test.ts b/tests/native/backup-archive-replacement.test.ts similarity index 74% rename from tests/native/snapshot-replacement.test.ts rename to tests/native/backup-archive-replacement.test.ts index 00500cd..4d9a98a 100644 --- a/tests/native/snapshot-replacement.test.ts +++ b/tests/native/backup-archive-replacement.test.ts @@ -4,9 +4,9 @@ import { join } from "node:path"; import { convexToJson } from "convex/values"; import { expect, onTestFinished, test } from "vitest"; import type { Backend } from "../../harness/backend.js"; -import { archiveEntries } from "./snapshot-archive.js"; +import { archiveEntries } from "./backup-archive-entries.js"; import { fixtureBackend, measure } from "../../harness/native.js"; -import { deploySnapshotFixture } from "../../harness/snapshot.js"; +import { deployTemporaryCopy } from "../../harness/backup-archive.js"; const root = join(import.meta.dirname, "../.."); @@ -16,19 +16,19 @@ async function schedulingComposition(directory: string) { await symlink(join(root, "node_modules"), join(directory, "node_modules")); await mkdir(join(directory, "tests/native"), { recursive: true }); await cp( - join(import.meta.dirname, "snapshot-scheduler.ts"), - join(directory, "tests/native/snapshot-scheduler.ts"), + join(import.meta.dirname, "backup-archive-scheduler.ts"), + join(directory, "tests/native/backup-archive-scheduler.ts"), ); await writeFile( - join(directory, "fixture/convex/snapshotSchedule.ts"), - 'export { schedule } from "../../tests/native/snapshot-scheduler.js";\n', + join(directory, "fixture/convex/archiveSchedule.ts"), + 'export { schedule } from "../../tests/native/backup-archive-scheduler.js";\n', ); } async function createDocuments(backend: Backend, prefix: string) { await backend.admin.run("depotRelay:createDocuments", { tenantId: "t-1", - actor: { kind: "operator", id: "snapshot-test" }, + actor: { kind: "operator", id: "backup-archive-test" }, operation: { operationId: prefix, causedBy: { kind: "command", commandType: "fixture" }, @@ -55,23 +55,23 @@ async function observe(backend: Backend) { }; } -test("native: snapshot replacement restores parent and context data and preserves destination configuration and schedules", async ({ +test("native: a backup archive imported with replacement restores parent and context data and preserves destination configuration and schedules", async ({ signal, }) => { - const directory = await mkdtemp(join(tmpdir(), "snapshot-test-")); + const directory = await mkdtemp(join(tmpdir(), "backup-archive-test-")); onTestFinished(() => rm(directory, { recursive: true, force: true })); await schedulingComposition(directory); const source = await fixtureBackend(); - const deployed = await deploySnapshotFixture(source, directory, signal); + const deployed = await deployTemporaryCopy(source, directory, signal); measure("fixture scheduling deployment command", { baseComposition: "fixture", - addedModule: "snapshotSchedule:schedule", + addedModule: "archiveSchedule:schedule", ...deployed, }); expect(source.facts().composition).toBeNull(); await createDocuments(source, "exported"); - await source.admin.setEnvironment({ SNAPSHOT_VALUE: "exported" }); - await source.admin.run("snapshotSchedule:schedule"); + await source.admin.setEnvironment({ ARCHIVE_VALUE: "exported" }); + await source.admin.run("archiveSchedule:schedule"); await expect .poll( async () => { @@ -82,11 +82,11 @@ test("native: snapshot replacement restores parent and context data and preserve ) .toEqual(["failed", "pending", "success"]); const before = await observe(source); - measure("snapshot source", convexToJson(before)); - const path = join(directory, "snapshot.zip"); - await source.admin.exportSnapshot(path); + measure("backup archive source", convexToJson(before)); + const path = join(directory, "backup-archive.zip"); + await source.admin.exportBackupArchive(path); const entries = await archiveEntries(path); - measure("snapshot archive entries", entries); + measure("backup archive entries", entries); for (const entry of [ "markers/documents.jsonl", "notes/documents.jsonl", @@ -102,9 +102,9 @@ test("native: snapshot replacement restores parent and context data and preserve const fresh = await fixtureBackend(); // A table the backend holds but does not list among the tables reads as empty, not as missing. expect(await fresh.admin.readTable("_scheduled_functions")).toEqual([]); - await fresh.admin.setEnvironment({ SNAPSHOT_VALUE: "destination" }); + await fresh.admin.setEnvironment({ ARCHIVE_VALUE: "destination" }); const freshEnvironment = await fresh.admin.environment(); - await fresh.admin.replaceSnapshot(path); + await fresh.admin.importBackupArchive(path); const freshAfter = await observe(fresh); measure("fresh replacement contents", convexToJson(freshAfter)); expect(freshAfter).toEqual({ @@ -114,7 +114,7 @@ test("native: snapshot replacement restores parent and context data and preserve }); await createDocuments(source, "after-export"); - await source.admin.run("snapshotSchedule:schedule"); + await source.admin.run("archiveSchedule:schedule"); await expect .poll( async () => { @@ -124,10 +124,10 @@ test("native: snapshot replacement restores parent and context data and preserve { timeout: 10000 }, ) .toEqual(["failed", "failed", "pending", "pending", "success", "success"]); - await source.admin.setEnvironment({ SNAPSHOT_VALUE: "after-export" }); + await source.admin.setEnvironment({ ARCHIVE_VALUE: "after-export" }); const sameBefore = await observe(source); measure("same backend before replacement", convexToJson(sameBefore)); - await source.admin.replaceSnapshot(path); + await source.admin.importBackupArchive(path); const sameAfter = await observe(source); measure("same backend replacement contents", convexToJson(sameAfter)); expect(sameAfter).toEqual({ diff --git a/tests/native/snapshot-scheduler.ts b/tests/native/backup-archive-scheduler.ts similarity index 96% rename from tests/native/snapshot-scheduler.ts rename to tests/native/backup-archive-scheduler.ts index 201b3e8..782abc6 100644 --- a/tests/native/snapshot-scheduler.ts +++ b/tests/native/backup-archive-scheduler.ts @@ -3,7 +3,7 @@ import { v } from "convex/values"; import { internal } from "../../fixture/convex/_generated/api.js"; import { internalMutation } from "../../fixture/convex/_generated/server.js"; -// Deployed only in a temporary copy of the fixture composition by the snapshot test. +// Deployed only in a temporary copy of the fixture composition by the backup archive test. export const schedule = internalMutation({ args: {}, returns: v.array(v.id("_scheduled_functions")), diff --git a/tests/native/scheduler-cases.ts b/tests/native/scheduler-cases.ts index 021f722..ce04ffc 100644 --- a/tests/native/scheduler-cases.ts +++ b/tests/native/scheduler-cases.ts @@ -6,7 +6,7 @@ import type { Value } from "convex/values"; import type { Backend } from "../../harness/backend.js"; import { measure } from "../../harness/native.js"; import { paceAfterWrite } from "../../harness/wait.js"; -import { archiveEntries } from "./snapshot-archive.js"; +import { archiveEntries } from "./backup-archive-entries.js"; import { initialize, schedulerRows, @@ -334,8 +334,8 @@ export async function exportData( const schedules = await schedulerRows(backend); record("exported data", data); record("exported schedules", schedules); - const path = join(directory, "snapshot.zip"); - await backend.admin.exportSnapshot(path); + const path = join(directory, "backup-archive.zip"); + await backend.admin.exportBackupArchive(path); const entries = await archiveEntries(path); measure("archive entries", entries); for (const entry of [ diff --git a/tests/native/scheduler-restore.ts b/tests/native/scheduler-restore.ts index 42f2dc9..e3fd0ba 100644 --- a/tests/native/scheduler-restore.ts +++ b/tests/native/scheduler-restore.ts @@ -15,7 +15,7 @@ export async function freshImport(backend: Backend, directory: string) { const { backend: fresh } = await schedulingBackend(); await fresh.admin.setEnvironment({ SCHEDULER_VALUE: "destination" }); const environment = await fresh.admin.environment(); - await fresh.admin.replaceSnapshot(exported.path); + await fresh.admin.importBackupArchive(exported.path); const restored = await dataRows(fresh); record("fresh restored data", restored); @@ -125,7 +125,7 @@ export async function inPlaceImport(backend: Backend, directory: string) { expect(changedData[table]).not.toEqual(exported.data[table]); const before = await schedulerRows(backend); record("in-place scheduler tables before import", before); - await backend.admin.replaceSnapshot(exported.path); + await backend.admin.importBackupArchive(exported.path); expect(Date.now()).toBeLessThan(due); const after = await schedulerRows(backend); record("in-place scheduler tables after import", after); diff --git a/tests/pure/child.test.ts b/tests/pure/child.test.ts index 04223b9..74c2ccf 100644 --- a/tests/pure/child.test.ts +++ b/tests/pure/child.test.ts @@ -314,7 +314,7 @@ test("pure: captured child failures carry the redacted command, exit code and la "process.stdout.write('stdout omitted'); for (let n = 1; n <= 25; n++) console.error(n + ':' + process.argv[1]); process.exit(7)", secret, ]; - const error = await runChild("snapshot child", process.execPath, args, { + const error = await runChild("archive child", process.execPath, args, { timeoutMs: 10000, secrets: [secret], output: "both", @@ -339,7 +339,7 @@ test("pure: captured child failures carry the redacted command, exit code and la test("pure: captured child output redacts both streams and abort rejects with the signal reason", async () => { expect( await runChild( - "snapshot child", + "archive child", process.execPath, [ "-e", @@ -354,9 +354,9 @@ test("pure: captured child output redacts both streams and abort rejects with th ), ).toEqual({ stdout: "[redacted]\n", stderr: "[redacted]\n" }); const controller = new AbortController(); - const reason = new Error("snapshot stopped"); + const reason = new Error("archive export stopped"); const result = runChild( - "snapshot child", + "archive child", process.execPath, ["-e", "setInterval(() => {}, 1000)"], { diff --git a/tests/pure/harness-rules.test.ts b/tests/pure/harness-rules.test.ts index 1e48036..476cc56 100644 --- a/tests/pure/harness-rules.test.ts +++ b/tests/pure/harness-rules.test.ts @@ -40,7 +40,7 @@ import { redactedBuffer } from "../../harness/child.js"; import { invocation } from "../../harness/evidence.js"; import { createAdminAccess } from "../../harness/admin.js"; import { fixtureComposition } from "../../harness/composition.js"; -import { deploySnapshotFixture } from "../../harness/snapshot.js"; +import { deployTemporaryCopy } from "../../harness/backup-archive.js"; import { localWriteRateBytesPerSecond, paceAfterWrite, @@ -225,15 +225,15 @@ test("pure: admin access reads an empty scheduled-functions table and refuses a ); }); -test("pure: the snapshot members use the pinned CLI and shared child runner with an isolated home", async () => { +test("pure: the backup archive members use the pinned CLI and shared child runner with an isolated home", async () => { const runner = vi .spyOn(child, "runChild") .mockResolvedValue({ stdout: "", stderr: "" }); try { const controller = new AbortController(); const admin = testAdmin(controller.signal); - expect(await admin.exportSnapshot("archive.zip")).toBeUndefined(); - expect(await admin.replaceSnapshot("archive.zip")).toBeUndefined(); + expect(await admin.exportBackupArchive("archive.zip")).toBeUndefined(); + expect(await admin.importBackupArchive("archive.zip")).toBeUndefined(); expect(runner).toHaveBeenCalledTimes(2); for (const [index, args] of [ ["export", "--path", "archive.zip"], @@ -270,11 +270,11 @@ test("pure: the snapshot members use the pinned CLI and shared child runner with } }); -test("pure: a rejected snapshot child retains the abort reason and removes its home", async () => { - const reason = new Error("snapshot aborted"); +test("pure: a rejected backup archive child retains the abort reason and removes its home", async () => { + const reason = new Error("archive export aborted"); const runner = vi.spyOn(child, "runChild").mockRejectedValue(reason); try { - await expect(testAdmin().exportSnapshot("archive.zip")).rejects.toBe( + await expect(testAdmin().exportBackupArchive("archive.zip")).rejects.toBe( reason, ); expect(existsSync(runner.mock.calls[0]![3].env!.HOME!)).toBe(false); @@ -302,7 +302,7 @@ test("pure: aborting the caller's signal stops the deploy child of a temporary d const backend = new AbortController(); const reason = new Error(`the ${owner} aborted the deploy`); let outcome: unknown = "pending"; - void deploySnapshotFixture( + void deployTemporaryCopy( { admin: testAdmin(backend.signal) }, "unused", caller.signal, From ba878de5ff35aea5c54d1d39d32faa463218c59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:41:45 +0200 Subject: [PATCH 118/155] build: 4f the compiled tier and the index The compiled tier is the one the doc's four tiers lack, not a fifth: the acceptance contract's rule and E-14 option, the two compiled examples, and the index rows E-13 and E-14 of design/README.md with its two notes take the new tier names. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 8 ++++---- ...eclaration.broken-argument-contract-fails-build.sdp.md | 2 +- ...command-declaration.renamed-handler-fails-build.sdp.md | 2 +- design/specs/platform/acceptance-contract.sdp.md | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/design/README.md b/design/README.md index 7659de1..84b9e0b 100644 --- a/design/README.md +++ b/design/README.md @@ -196,8 +196,8 @@ An extension is a design claim the doc does not make. Each one is three things t | E-10 | [`obligations.lifecycle-transitions`](specs/obligations/lifecycle-transitions.sdp.md), [`obligations.record-contract`](specs/obligations/record-contract.sdp.md) | The allowed transitions between the six states and who causes each; the `obligations` and `obligationRepairs` tables with seven and two indexes; a 64 KiB inline payload bound; ten late-evidence entries; two attention reasons beyond D13's four, `recoveryFailing` and `chainBound` | | E-11 | [`effects.claim-call-settle`](specs/effects/claim-call-settle.sdp.md) | The signatures of `claimAttempt`, `readForCall`, `callProvider`, `reconcileAttempt`, `settleAttempt`, `expireLease`; the `RepetitionPolicy` union with a fourth value `none` | | E-12 | [`constraints.events-stay-small`](specs/constraints/events-stay-small.sdp.md), [`context.event-envelope`](specs/context/event-envelope.sdp.md) | A 16 KiB payload bound and a 4,096-byte envelope bound measured with `getConvexSize` by the journal's `append`, so a fold of 500 stored events of at most 20,541 bytes each stays under 9.8 MiB under the 16 MiB read ceiling; the reserved `baseline` event's payload is exempt and bounded by its stream type's `budgetBytes`, and the stored baseline event is at most `budgetBytes + 4,096 + 61` bytes; the 1 second query timeout binds first and the first experiment measures it | -| E-13 | [`platform.native-harness`](specs/platform/native-harness.sdp.md) | The native tier's callers are ordinary clients carrying tokens the harness signs for a fixture issuer; a composition that runs natively reads its issuer, application ID and a data-URI key set from the deployment's environment variables, which reaches into the production `auth.config.ts`; parity for Sc L2-9 is code path, schema, concurrency, the authority model and Convex's visibility check; the issuer is the identity source and the one named difference on every evidence record. The admin key acting as an identity was dropped on 2026-10-01: it passes the visibility check for internal functions | -| E-14 | [`platform.acceptance-contract`](specs/platform/acceptance-contract.sdp.md) | A fifth tier, the build tier: a type test of a composition's generated `api`, a typed caller that states the change and must fail `tsc`, run by the type check with no backend; Sc L2-4 is its one scenario | +| E-13 | [`platform.native-harness`](specs/platform/native-harness.sdp.md) | The native backend tier's callers are ordinary clients carrying tokens the harness signs for a fixture issuer; a composition that runs natively reads its issuer, application ID and a data-URI key set from the deployment's environment variables, which reaches into the production `auth.config.ts`; parity for Sc L2-9 is code path, schema, concurrency, the authority model and Convex's visibility check; the issuer is the identity source and the one named difference on every evidence record. The admin key acting as an identity was dropped on 2026-10-01: it passes the visibility check for internal functions | +| E-14 | [`platform.acceptance-contract`](specs/platform/acceptance-contract.sdp.md) | The compiled tier, which the doc's four tiers lack: a type test of a composition's generated `api`, a typed caller that states the change and must fail `tsc`, run by the type check with no backend; Sc L2-4 is its one scenario | | E-15 | [`platform.native-harness`](specs/platform/native-harness.sdp.md) | The native harness: the admin key is for setup and reading, never for a caller under test; two compositions, fixture and production, each a name, a project directory and a functions directory, the production composition's project directory being `example/`, with the generated files of both committed; faults only through what the fixture composition supplies, and no test hook in the shared libraries; assertions only on what a client or admin access can read, what a command read and wrote taken from the `usageStats` of its top-level completion record, and a refusal shown with its own error, nothing stored and the refused thing existing, shown by an ordinary client where one may call it and with admin access otherwise; the fixture action `nonUiCaller:send` as the fixture composition's stand-in for trusted server code; four controls (the backend process, the clients, the environment variables, fixture functions) and no clock or scheduler switch; the production-configuration run of Sc L2-9 is the scenarios routed to the production composition; a pass claim cites a record kept in the repository | | E-16 | [`facts.probe-plan`](specs/facts/probe-plan.sdp.md) | How a probe is written and recorded: example Specs that verify the fact, bound values as the expectation written before the first run, a different first result recorded on the fact and the example then bound to what the release does, a run that must reach its boundary, sizes recorded and not asserted, and what a local backend cannot answer left open on the fact | | E-17 | [`platform.acceptance-contract`](specs/platform/acceptance-contract.sdp.md), [`application.first-experiment`](specs/application/first-experiment.sdp.md) | The acceptance check: the required scenarios generated from the graph, one per example of a required row with its tier, composition and verifiers, read against one run's record; a scenario is passed, failed, absent or missing, and the whole passes only when every one passed on a record of a passed run on a clean tree; `acceptanceRows` of `application.first-experiment` names the first experiment's twenty-three rows | @@ -401,7 +401,7 @@ Cross-package fixes made in review round 2: - History views. `HistoryProjection` folds events, the adapter's `streams` entry carries the appended events, the rebuild's pause path pages each source's history, and the example domain has the order allocation history view that both Sc L2-6 examples build. - The write pause's scope. A paused rebuild closes the view's source stream types, and the declaration's `writes` lets the pipeline hold every writer of them. - The example domain. The stock item stream declares a 16 KiB budget so the maximum order fits one allocation call, and `PlaceOrder` calls Orders first so a double submit is answered `entityExists`. -- The build tier is the acceptance contract's fifth tier (E-14), and the three Layer 4 to 6 Specs that lacked a `deferred:` Design bullet have one. +- The compiled tier, which the doc's four tiers lack, is the acceptance contract's own (E-14), and the three Layer 4 to 6 Specs that lacked a `deferred:` Design bullet have one. Things in the plan that the packages found wrong, and the convention adopted: @@ -410,7 +410,7 @@ Things in the plan that the packages found wrong, and the convention adopted: - Unnumbered sections of the doc are cited by name (the last row of the numbering key); the plan's section 1 should list them. - The composition helper returns two static exports per command, not one; a public mutation cannot take a server-assigned namespace without letting a client claim it (Sc L1-7). - The receipt sweep is per tenant (`by_tenant_expiry`), because rubric item 25 forbids a non-tenant-leading index; the obligation module's three sweeper and retention indexes break that rule deliberately and record it as E-57. The `generations` table also has no `tenantId` lead, because a generation spans all tenants; the registry's E-8 question on the reach of a generation records that deviation. -- Sc L2-4 runs in a build tier the doc's four tiers do not name; the acceptance contract now carries it as a fifth tier under E-14. +- Sc L2-4 runs in the compiled tier, which the doc's four tiers lack; the acceptance contract now carries it under E-14. - Layer 4 to 6 deferrals were `[non-blocking]` open questions until 2026-10-01. Each is now a `[blocking]` question that names its trigger, which is the Protocol's home for a deferral and what holds the Spec at `scoped`. - A `contract` Spec clears `defined` through its Contract bullets; no example children are needed. - Package B's pack lists each parent's examples directly after it, as do the others. diff --git a/design/specs/command/command-declaration.broken-argument-contract-fails-build.sdp.md b/design/specs/command/command-declaration.broken-argument-contract-fails-build.sdp.md index 6877d64..4d11f4f 100644 --- a/design/specs/command/command-declaration.broken-argument-contract-fails-build.sdp.md +++ b/design/specs/command/command-declaration.broken-argument-contract-fails-build.sdp.md @@ -27,6 +27,6 @@ And it fails {when: "before any traffic"} ## Verification — executable -- Runs in the compiled tier, the fifth tier `spec:platform.acceptance-contract` adds under E-14, with no backend, as a type test of the production composition's generated `api`. +- Runs in the compiled tier, which the doc's four tiers lack and `spec:platform.acceptance-contract` adds under E-14, with no backend, as a type test of the production composition's generated `api`. - The binding is a type test: the compiler is the check, and no step body runs. - The test asserts that a call of `api.ordering.placeOrder` whose input lacks a required field, and one whose field has another type, each fail `tsc` through the compiler's expected-error directive. diff --git a/design/specs/command/command-declaration.renamed-handler-fails-build.sdp.md b/design/specs/command/command-declaration.renamed-handler-fails-build.sdp.md index ef21452..797ea74 100644 --- a/design/specs/command/command-declaration.renamed-handler-fails-build.sdp.md +++ b/design/specs/command/command-declaration.renamed-handler-fails-build.sdp.md @@ -27,6 +27,6 @@ And it fails {when: "before any traffic"} ## Verification — executable -- Runs in the compiled tier, the fifth tier `spec:platform.acceptance-contract` adds under E-14, with no backend, as a type test of the production composition's generated `api`. +- Runs in the compiled tier, which the doc's four tiers lack and `spec:platform.acceptance-contract` adds under E-14, with no backend, as a type test of the production composition's generated `api`. - The binding is a type test: the compiler is the check, and no step body runs. - The test asserts that `api.ordering.placeOrder` is a public mutation reference, that a reference to a name the module does not export fails `tsc`, and that a typed call of `placeOrder` through the api built, as the generated `api` is, from the module with that export renamed `submitOrder` fails `tsc`, each through the compiler's expected-error directive, which itself fails the compile when the line stops being an error. diff --git a/design/specs/platform/acceptance-contract.sdp.md b/design/specs/platform/acceptance-contract.sdp.md index 5d8182a..928e5db 100644 --- a/design/specs/platform/acceptance-contract.sdp.md +++ b/design/specs/platform/acceptance-contract.sdp.md @@ -21,7 +21,7 @@ The doc's acceptance table is the contract. It lists forty-three scenario rows b ### Open questions -- [non-blocking] Extension E-14: the doc's four tiers, domain, simulator, native and end to end, name no compile-time check, while Sc L2-4's pass condition is a build, type or registration check that fails before any traffic; the option taken here is a fifth tier, the compiled tier, a compile, with no backend, of a typed caller that states the change, which asserts that `tsc` refuses it, run with every type check and recorded like any run; the alternative, folding it into the native backend tier as a compile step that precedes the native run, was not taken because a native run needs a backend and the check needs none; the owner confirms (E-14, Sc L2-4, Acceptance scenarios) +- [non-blocking] Extension E-14: the doc's four tiers, domain, simulator, native and end to end, name no compile-time check, while Sc L2-4's pass condition is a build, type or registration check that fails before any traffic; the option taken here is the compiled tier, a compile, beside the doc's four, of which this corpus reads the end to end tier as the native backend tier on the production composition, with no backend, of a typed caller that states the change, which asserts that `tsc` refuses it, run with every type check and recorded like any run; the alternative, folding it into the native backend tier as a compile step that precedes the native run, was not taken because a native run needs a backend and the check needs none; the owner confirms (E-14, Sc L2-4, Acceptance scenarios) - [non-blocking] Extension E-17: the doc says that a link between source and test is not a passing run and that the first experiment passes when every Layer 0, 1 and 2 scenario passes, and it does not say how a pass of the whole is established from runs; the option taken here is the acceptance check: the required scenarios are generated from the graph, one per example of a required row with its tier, composition and verifiers, and read against one run's record, so that a scenario with no verifier, with no result or with a result other than passed keeps the whole from passing; the alternative, a list of scenarios and verifiers kept by hand beside the tests, was not taken because the graph already holds each example's row, tier, composition and verifiers; the owner confirms (E-17, Acceptance scenarios, First experiment) ## Rule @@ -43,7 +43,7 @@ The doc's acceptance table is the contract. It lists forty-three scenario rows b - Readiness is stated plainly as specified, implemented, tested under named conditions, or operationally accepted (Acceptance scenarios) - The first experiment passes when every Layer 0, 1 and 2 scenario passes on a native backend (First experiment) - The one all-layer scenario, breaking metrics and logging and separately breaking mandatory audit, applies to every installed layer (Sc ALL-1) -- [extension] A compiled-tier scenario passes when a typed caller that states the change fails `tsc` before any deployment; the compiled tier is the fifth tier, named by Sc L2-4 alone, and needs no backend (E-14, Sc L2-4) +- [extension] A compiled-tier scenario passes when a typed caller that states the change fails `tsc` before any deployment; the compiled tier, which the doc's four tiers lack, is named by Sc L2-4 alone and needs no backend (E-14, Sc L2-4) - [extension] Three statements about a scenario stay apart: it is bound when a verifier of its example resolves in the graph, it passed when a run's record holds a passing result of that verifier, and a set of scenarios is accepted only when the acceptance check finds every one of them passed in one record (E-17, Acceptance scenarios) - [extension] No example and no test stands for another scenario: a bound step names what its own test runs, and a statement about every scenario of a layer is the acceptance check's to make (E-17, Acceptance scenarios) - [extension] The acceptance check takes the rows a Spec requires and one run's record, derives its required scenarios from the graph, and passes only when every required scenario passed (E-17, First experiment) From 14f93ce3040a4bb1fc2cbd0941d526e25ca5b01d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:42:05 +0200 Subject: [PATCH 119/155] design: 4f E-14's option reads as one clause Co-Authored-By: Claude Opus 5.5 --- design/specs/platform/acceptance-contract.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/platform/acceptance-contract.sdp.md b/design/specs/platform/acceptance-contract.sdp.md index 928e5db..68c21ce 100644 --- a/design/specs/platform/acceptance-contract.sdp.md +++ b/design/specs/platform/acceptance-contract.sdp.md @@ -21,7 +21,7 @@ The doc's acceptance table is the contract. It lists forty-three scenario rows b ### Open questions -- [non-blocking] Extension E-14: the doc's four tiers, domain, simulator, native and end to end, name no compile-time check, while Sc L2-4's pass condition is a build, type or registration check that fails before any traffic; the option taken here is the compiled tier, a compile, beside the doc's four, of which this corpus reads the end to end tier as the native backend tier on the production composition, with no backend, of a typed caller that states the change, which asserts that `tsc` refuses it, run with every type check and recorded like any run; the alternative, folding it into the native backend tier as a compile step that precedes the native run, was not taken because a native run needs a backend and the check needs none; the owner confirms (E-14, Sc L2-4, Acceptance scenarios) +- [non-blocking] Extension E-14: the doc's four tiers, domain, simulator, native and end to end, name no compile-time check, while Sc L2-4's pass condition is a build, type or registration check that fails before any traffic; the option taken here is the compiled tier, a compile with no backend of a typed caller that states the change, which asserts that `tsc` refuses it, run with every type check and recorded like any run, and this corpus reads the doc's end to end tier as the native backend tier on the production composition; the alternative, folding it into the native backend tier as a compile step that precedes the native run, was not taken because a native run needs a backend and the check needs none; the owner confirms (E-14, Sc L2-4, Acceptance scenarios) - [non-blocking] Extension E-17: the doc says that a link between source and test is not a passing run and that the first experiment passes when every Layer 0, 1 and 2 scenario passes, and it does not say how a pass of the whole is established from runs; the option taken here is the acceptance check: the required scenarios are generated from the graph, one per example of a required row with its tier, composition and verifiers, and read against one run's record, so that a scenario with no verifier, with no result or with a result other than passed keeps the whole from passing; the alternative, a list of scenarios and verifiers kept by hand beside the tests, was not taken because the graph already holds each example's row, tier, composition and verifiers; the owner confirms (E-17, Acceptance scenarios, First experiment) ## Rule From 221a5e1ed9430421e8597dac96bce896b4aa7ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:42:42 +0200 Subject: [PATCH 120/155] build: hosted redaction at the child and the admin boundary A child's output is redacted on success in both modes, and an abort reason that carries a secret is replaced by a redacted error. Every member of the admin access passes its rejection through redaction where it is first seen, a fetch that rejects included, and its HTTP and socket clients log through a redacting logger. Co-Authored-By: Claude Opus 5.5 --- harness/admin.ts | 38 +++++++++++++--- harness/child.ts | 75 +++++++++++++++++++++++++++++++- tests/pure/child.test.ts | 36 +++++++++++++++ tests/pure/harness-rules.test.ts | 61 ++++++++++++++++++++++++++ 4 files changed, 203 insertions(+), 7 deletions(-) diff --git a/harness/admin.ts b/harness/admin.ts index 22abed5..e02b600 100644 --- a/harness/admin.ts +++ b/harness/admin.ts @@ -9,7 +9,7 @@ import type { Value } from "convex/values"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; -import { redact, runChild } from "./child.js"; +import { redact, redactError, redactingLogger, runChild } from "./child.js"; import { isolatedCli, createBackupArchiveAccess } from "./backup-archive.js"; import type { CliOutput } from "./backup-archive.js"; import { projectDirectory } from "./composition.js"; @@ -149,7 +149,10 @@ export function createAdminAccess( state: AdminState, ): AdminAccess { const marks = new WeakMap(); - const http = new ConvexHttpClient(target.url) as AdminHttpClient; + const logger = redactingLogger(target.secrets); + const http = new ConvexHttpClient(target.url, { + logger, + }) as AdminHttpClient; http.setAdminAuth(target.adminKey); const authorization = { Authorization: `Convex ${target.adminKey}` }; async function cli(command: "deploy" | "codegen") { @@ -191,7 +194,7 @@ export function createAdminAccess( if (value !== undefined) return settle(() => resolve(value)); } }, - { unsavedChangesWarning: false }, + { unsavedChangesWarning: false, logger }, ) as unknown as AdminSocketClient; const timer = setTimeout( () => @@ -210,7 +213,7 @@ export function createAdminAccess( ); }); } - return { + return guarded(target.secrets, { ...createBackupArchiveAccess(target), async deploy() { await cli("deploy"); @@ -394,5 +397,30 @@ export function createAdminAccess( assertProcess(); return records; }, - }; + }); +} +// Every member's rejection, and anything one throws at once, passes through redaction where it is +// first seen, so no secret leaves the admin access in an error. +export function guarded( + secrets: readonly string[], + access: T, +): T { + const result: Record = {}; + for (const [name, member] of Object.entries(access)) + result[name] = + typeof member === "function" + ? (...args: unknown[]) => { + try { + const value: unknown = member.apply(access, args); + return value instanceof Promise + ? value.catch((error: unknown) => { + throw redactError(error, secrets); + }) + : value; + } catch (error) { + throw redactError(error, secrets); + } + } + : member; + return result as T; } diff --git a/harness/child.ts b/harness/child.ts index a65c2bc..3f81211 100644 --- a/harness/child.ts +++ b/harness/child.ts @@ -4,6 +4,7 @@ import { ref, } from "@libar-dev/software-delivery-protocol"; import { execFile } from "node:child_process"; +import { format } from "node:util"; const anchor = codeAnchor({ id: codeAnchorId("impl:platform.native-harness.child"), label: "child processes and redaction", @@ -36,6 +37,73 @@ export function redact(text: string, secrets: readonly string[]): string { } return result; } +// Everything of an error that a reporter or a log can print: its name, message and stack, its own +// properties and its causes. +function printable(error: unknown, depth = 0): string { + const text = (value: unknown) => { + try { + return typeof value === "string" ? value : String(value); + } catch { + return ""; + } + }; + const json = (value: unknown) => { + try { + return JSON.stringify(value) ?? ""; + } catch { + return text(value); + } + }; + if (!(error instanceof Error)) return `${text(error)}\n${json(error)}`; + const parts = [error.name, error.message, error.stack ?? ""]; + for (const [key, value] of Object.entries(error)) + parts.push(key, text(value), json(value)); + if (error.cause !== undefined && depth < 3) + parts.push(printable(error.cause, depth + 1)); + return parts.join("\n"); +} +// An error that carries no secret anywhere it can be printed is returned as it is, so that a caller +// can still tell it by identity. One that carries a secret is replaced by an Error of the same name +// whose message and stack have every secret replaced, and which keeps none of its properties. +export function redactError( + error: unknown, + secrets: readonly string[], +): unknown { + const keys = secrets.filter((secret) => secret !== ""); + if (keys.length === 0) return error; + const printed = printable(error); + if (!keys.some((key) => printed.includes(key))) return error; + const original = error instanceof Error ? error : undefined; + const replaced = new Error(redact(original?.message ?? String(error), keys)); + if (original !== undefined) { + replaced.name = original.name; + replaced.stack = redact( + original.stack ?? `${original.name}: ${original.message}`, + keys, + ); + } + return replaced; +} +// A logger for Convex's clients, which log a function's log lines and their own warnings, with +// every secret replaced before it reaches the console. +export interface ClientLogger { + logVerbose(...args: unknown[]): void; + log(...args: unknown[]): void; + warn(...args: unknown[]): void; + error(...args: unknown[]): void; +} +export function redactingLogger(secrets: readonly string[]): ClientLogger { + const write = + (method: "debug" | "log" | "warn" | "error") => + (...args: unknown[]) => + console[method](redact(format(...args), secrets)); + return { + logVerbose: write("debug"), + log: write("log"), + warn: write("warn"), + error: write("error"), + }; +} export interface ChildOutput { stdout: string; stderr: string; @@ -78,9 +146,12 @@ export function runChild( stdout: redact(stdout, options.secrets ?? []), stderr: redact(stderr, options.secrets ?? []), } - : stdout, + : redact(stdout, options.secrets ?? []), + ); + if (options.signal?.aborted) + return reject( + redactError(options.signal.reason, options.secrets ?? []), ); - if (options.signal?.aborted) return reject(options.signal.reason); if (options.output === "both") return reject( new Error( diff --git a/tests/pure/child.test.ts b/tests/pure/child.test.ts index 74c2ccf..c812d93 100644 --- a/tests/pure/child.test.ts +++ b/tests/pure/child.test.ts @@ -368,3 +368,39 @@ test("pure: captured child output redacts both streams and abort rejects with th controller.abort(reason); await expect(result).rejects.toBe(reason); }); + +test("pure: a succeeded child's output is redacted in either output mode", async () => { + const args = [ + "-e", + "console.log(process.argv[1]); console.error(process.argv[1])", + secret, + ]; + expect( + await runChild("the child", process.execPath, args, { + timeoutMs: 10000, + secrets: [secret], + }), + ).toBe("[redacted]\n"); +}); + +test("pure: an abort reason that carries a secret is redacted, and one that carries none is returned as it is", async () => { + for (const carries of [true, false]) { + const controller = new AbortController(); + const reason = new Error(carries ? `stopped ${secret}` : "stopped"); + const result = runChild( + "the child", + process.execPath, + ["-e", "setInterval(() => {}, 1000)"], + { timeoutMs: 10000, signal: controller.signal, secrets: [secret] }, + ); + controller.abort(reason); + const error = await result.catch((thrown: unknown) => thrown); + if (!carries) { + expect(error).toBe(reason); + continue; + } + expect(error).not.toBe(reason); + expect((error as Error).message).toBe("stopped [redacted]"); + expect((error as Error).stack).not.toContain(secret); + } +}); diff --git a/tests/pure/harness-rules.test.ts b/tests/pure/harness-rules.test.ts index 476cc56..cd5284c 100644 --- a/tests/pure/harness-rules.test.ts +++ b/tests/pure/harness-rules.test.ts @@ -319,3 +319,64 @@ test("pure: aborting the caller's signal stops the deploy child of a temporary d runner.mockRestore(); } }); + +test("pure: every admin member's rejection and every client log line has its secrets replaced, a fetch that rejects included", async () => { + const key = "REVIEW_DUMMY_ADMIN_SECRET_0123456789"; + const admin = createAdminAccess( + { + url: "http://127.0.0.1:1", + adminKey: key, + home: "unused", + composition: fixtureComposition, + secrets: [key], + }, + { deployed: undefined, environment: new Set(), logProcess: {} }, + ); + const printed: string[] = []; + const consoles = (["log", "warn", "error", "debug", "info"] as const).map( + (method) => + vi + .spyOn(console, method) + .mockImplementation((...args: unknown[]) => + printed.push(args.map(String).join(" ")), + ), + ); + try { + vi.stubGlobal("fetch", async () => { + throw new TypeError(`fetch failed for ${key}`); + }); + const failures: unknown[] = []; + for (const call of [ + () => admin.setEnvironment({ NAME: "value" }), + () => admin.run("notes:add"), + async () => admin.completionsSince(await admin.logMark(), () => false), + ]) + failures.push(await call().catch((error: unknown) => error)); + vi.stubGlobal( + "fetch", + async () => + new Response( + JSON.stringify({ + status: "success", + value: null, + logLines: [`[LOG] ${key}`], + }), + ), + ); + await admin.run("notes:add"); + expect(failures).toHaveLength(3); + for (const failure of failures) { + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain("[redacted]"); + } + const seen = JSON.stringify([ + failures.map((failure) => [String(failure), (failure as Error).stack]), + printed, + ]); + expect(printed.join("\n")).toContain("[redacted]"); + expect(seen).not.toContain(key); + } finally { + vi.unstubAllGlobals(); + for (const spy of consoles) spy.mockRestore(); + } +}); From d76fbd335059adee9739524779ad944cfbd4921f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:45:25 +0200 Subject: [PATCH 121/155] build: hosted the record's target and the acceptance filter A run record names its target apart from its tier, local backend or hosted deployment, and a run on a hosted deployment writes hosted- with its hosted facts, its tests under the project hosted and the deploys they stored. The reporter and an amendment redact the serialized record. The acceptance check reads only a record whose target is absent or the local backend. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- evidence/README.md | 4 +- harness/acceptance.ts | 11 ++- harness/backend.ts | 7 +- harness/evidence.ts | 109 ++++++++++++++++++---- tests/pure/evidence.test.ts | 4 + tests/pure/hosted.test.ts | 175 ++++++++++++++++++++++++++++++++++++ 7 files changed, 292 insertions(+), 20 deletions(-) create mode 100644 tests/pure/hosted.test.ts diff --git a/design/README.md b/design/README.md index 7a31223..c70aef1 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -237 specs · 5 packs · 131 anchors → 373 nodes · 1235 edges (0 errors, 0 warnings) +237 specs · 5 packs · 132 anchors → 374 nodes · 1236 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/evidence/README.md b/evidence/README.md index 473b4be..6e65e1f 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -10,6 +10,7 @@ A record that backs a claim says `"clean": true`. It was made on a tree with no - `command`, `startedAt`, `finishedAt` and `result`. - `versions`: Node and the packages the tiers depend on, read from the installed packages. - `tests`: one entry per test, with its result, its error messages, its backends and its measurements. A run that includes the native project records the tests of every project it ran, each with its `project`: `types`, `pure`, `simulator` or `native`. +- `target`: `local backend`, or `hosted deployment` for a run of `npm run test:hosted`, which is the native backend tier on a hosted deployment and not a tier of its own. Such a record is named `hosted-` and its start time, its tests are of the project `hosted`, and its `hosted` facts name the deployment, the deploy key by its name, the stated plan, who ran it, the CLI and backend versions, the usage readings before and after, and each deploy's wall time. The acceptance check reads no such record. A record with no `target` is of the local backend. A test's `backends` has one entry per backend that became ready. The harness writes the entry from the backend that ran: @@ -17,7 +18,8 @@ A test's `backends` has one entry per backend that became ready. The harness wri - `identitySource`: the fixture issuer that the backend's environment variables named. The fixture issuer is the one difference from production in how a caller gets an identity. It is not adjusted configuration. - `composition` and `installedLayers`: what the harness deployed. - `environment`: the names of the environment variables the harness set. -- `dataset`: every backend starts on empty storage, so a test's data is what the test wrote. +- `dataset`: every local backend starts on empty storage, so a test's data is what the test wrote. A hosted deployment keeps its data between runs, and says `kept from earlier runs`. +- `target`: the local backend or a hosted deployment. On a hosted deployment `executable` is `null`. ## Measurements diff --git a/harness/acceptance.ts b/harness/acceptance.ts index 129425c..fd8fe66 100644 --- a/harness/acceptance.ts +++ b/harness/acceptance.ts @@ -325,6 +325,10 @@ function hasRunRecordShape(value: unknown): value is Record { Array.isArray(value.tests) ); } +// A native run on a hosted deployment passes no scenario, so the check reads only a record whose +// target is absent or the local backend. +const targetsLocalBackend = (value: Record) => + value.target === undefined || value.target === "local backend"; // Why a test entry cannot be read, or null when it can. function entryProblem(entry: unknown): string | null { if (!isObject(entry)) return "is not an object"; @@ -358,6 +362,10 @@ function readRecord(path: string): AcceptanceRecord { } if (!hasRunRecordShape(record)) throw new AcceptanceUnanswerable(`The file ${path} is not a run record`); + if (!targetsLocalBackend(record)) + throw new AcceptanceUnanswerable( + `The record ${path} is not a run on the local backend, and a native run on a hosted deployment passes no scenario`, + ); for (const [index, entry] of (record.tests as unknown[]).entries()) { const problem = entryProblem(entry); if (problem !== null) @@ -388,7 +396,8 @@ export function newestRecord(directory: string): string { } catch { continue; } - if (!hasRunRecordShape(candidate)) continue; + if (!hasRunRecordShape(candidate) || !targetsLocalBackend(candidate)) + continue; const { startedAt } = candidate; if (typeof startedAt !== "string") continue; if ( diff --git a/harness/backend.ts b/harness/backend.ts index 94b3633..35c7c27 100644 --- a/harness/backend.ts +++ b/harness/backend.ts @@ -25,13 +25,15 @@ const anchor = codeAnchor({ satisfies: ref("spec:platform.native-harness"), }); void anchor; +// On a hosted deployment the executable is unknown and the data is kept from earlier runs. export interface BackendFacts { composition: Composition["name"] | null; installedLayers: string[]; - executable: Pick; + target: "local backend" | "hosted deployment"; + executable: Pick | null; identitySource: { kind: "fixture issuer"; issuer: string }; environment: string[]; - dataset: "empty at start"; + dataset: "empty at start" | "kept from earlier runs"; } export interface StartOptions { executable: Executable; @@ -293,6 +295,7 @@ export async function startBackend(options: StartOptions): Promise { facts: () => ({ composition: state.deployed?.name ?? null, installedLayers: [...(state.deployed?.installedLayers ?? [])], + target: "local backend", executable: { release: options.executable.release, sha256: options.executable.sha256, diff --git a/harness/evidence.ts b/harness/evidence.ts index 278135d..c89f934 100644 --- a/harness/evidence.ts +++ b/harness/evidence.ts @@ -10,6 +10,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import type { Reporter, TestCase } from "vitest/node"; import type { BackendFacts } from "./backend.js"; +import { redact } from "./child.js"; const anchor = codeAnchor({ id: codeAnchorId("impl:platform.native-harness.evidence"), label: "the run record", @@ -27,13 +28,44 @@ export interface NativeTestFacts { measurements: Measurement[]; } // The Vitest project that ran a test. The acceptance check reads it to confirm a scenario's tier. -export type TestProjectName = "types" | "pure" | "simulator" | "native"; +// `hosted` is the project of a test that ran on a hosted deployment. +export type TestProjectName = + "types" | "pure" | "simulator" | "native" | "hosted"; const projectNames: readonly string[] = [ "types", "pure", "simulator", "native", + "hosted", ]; +// Where a native run ran, recorded apart from the tier: a hosted deployment is a target of the +// native backend tier, not a tier of its own. +export type Target = "local backend" | "hosted deployment"; +// The deployment's answer to GET /api/v1/get_current_usage, with the time it was read. +export interface HostedUsage { + readAt: string; + seedStatus: string | null; + response: JsonValue; +} +export interface HostedDeploy { + composition: "fixture" | "production" | null; + wallMs: number; +} +export interface HostedRunFacts { + deployment: string; + keyName: string; + statedPlan: string; + ranBy: "continuous integration" | "developer"; + cliVersion: string; + backendVersion: string | null; + usageBefore: HostedUsage | null; + usageAfter: HostedUsage | null; + windowCrossed: boolean; + deploys: HostedDeploy[]; +} +// The measurement a test on a hosted deployment stores for each deploy, which the record gathers +// into its hosted facts. +export const hostedDeployMeasurement = "hosted deploy"; export interface NativeTestEntry extends NativeTestFacts { project: TestProjectName; name: string; @@ -44,6 +76,7 @@ export interface NativeTestEntry extends NativeTestFacts { } export interface NativeRunRecord { tier: "native"; + target: Target; commit: string; clean: boolean; command: string; @@ -53,6 +86,7 @@ export interface NativeRunRecord { versions: Record; tests: NativeTestEntry[]; unhandledErrors: string[]; + hosted: HostedRunFacts | null; } declare module "vitest" { interface TaskMeta { @@ -86,6 +120,7 @@ export default class EvidenceReporter implements Reporter { this.directory = options.directory ?? join(repositoryRoot, "evidence/runs"); } private native = false; + private hostedRun = false; private startedAt = ""; private commit = "unknown"; private clean = false; @@ -93,9 +128,14 @@ export default class EvidenceReporter implements Reporter { onTestRunStart( specifications: Parameters>[0], ) { - this.native = specifications.some( - (specification) => specification.project.name === "native", + this.hostedRun = specifications.some( + (specification) => specification.project.name === "hosted", ); + this.native = + this.hostedRun || + specifications.some( + (specification) => specification.project.name === "native", + ); this.startedAt = new Date().toISOString(); this.commit = git("rev-parse", "HEAD") ?? "unknown"; this.clean = git("status", "--porcelain") === ""; @@ -136,8 +176,10 @@ export default class EvidenceReporter implements Reporter { ), ) as { version: string } ).version; + const run = onProcess[runRecordKey]; const record: NativeRunRecord = { tier: "native", + target: this.hostedRun ? "hosted deployment" : "local backend", commit: this.commit, clean: this.clean, command: invocation(process.argv.slice(2)), @@ -147,12 +189,16 @@ export default class EvidenceReporter implements Reporter { versions, tests: this.tests, unhandledErrors: unhandledErrors.map((error) => error.message), + hosted: + this.hostedRun && run?.hosted !== undefined + ? { ...run.hosted, deploys: deploysOf(this.tests) } + : null, }; const directory = this.directory; await mkdir(directory, { recursive: true }); - const name = await writeRunRecord(directory, record); + const name = await writeRunRecord(directory, record, run?.secrets ?? []); rememberRunRecord(join(directory, name)); - console.log(`Native run record: evidence/runs/${name}`); + console.log(`Native run record (${record.target}): evidence/runs/${name}`); } } @@ -164,16 +210,35 @@ export function invocation(args: readonly string[]): string { return ["vitest", ...args].map(quote).join(" "); } +// The deploys that the tests of a run on a hosted deployment stored, in the order they ran. +function deploysOf(tests: readonly NativeTestEntry[]): HostedDeploy[] { + return tests.flatMap((test) => + test.measurements + .filter((measurement) => measurement.name === hostedDeployMeasurement) + .map((measurement) => measurement.value as unknown as HostedDeploy), + ); +} + +// The record serialized with every secret replaced, a second net behind the redaction where each +// response is first seen. +export function serializeRunRecord( + record: NativeRunRecord, + secrets: readonly string[] = [], +): string { + return redact(JSON.stringify(record, null, 2), secrets) + "\n"; +} + +// A run on a hosted deployment writes its record as hosted- and its start time. export async function writeRunRecord( directory: string, record: NativeRunRecord, + secrets: readonly string[] = [], ): Promise { - const name = `native-${record.startedAt.replace(/[-:]|\.\d+/g, "")}-${record.commit.slice(0, 7)}-${randomUUID()}.json`; - await writeFile( - join(directory, name), - JSON.stringify(record, null, 2) + "\n", - { flag: "wx" }, - ); + const prefix = record.target === "hosted deployment" ? "hosted" : "native"; + const name = `${prefix}-${record.startedAt.replace(/[-:]|\.\d+/g, "")}-${record.commit.slice(0, 7)}-${randomUUID()}.json`; + await writeFile(join(directory, name), serializeRunRecord(record, secrets), { + flag: "wx", + }); return name; } @@ -184,6 +249,10 @@ export async function writeRunRecord( export interface RunRecordSlot { path?: string; swept?: boolean; + // What a run on a hosted deployment knows before its tests: the secrets the record is redacted + // by and its hosted facts, without the deploys its tests store. + secrets?: readonly string[]; + hosted?: Omit; } const runRecordKey = Symbol.for("libar-platform.native-run-record"); const onProcess = globalThis as Record; @@ -200,14 +269,24 @@ export function rememberRunRecord(path: string): void { export function failRunRecord( run: RunRecordSlot, message: string, +): string | undefined { + return amendRunRecord(run, (record) => { + record.result = "failed"; + if (!record.unhandledErrors.includes(message)) + record.unhandledErrors.push(message); + }); +} +// Changes the run's record and replaces the old record in one step, redacted as it was written. +// Returns the record's path, if the run has one. +export function amendRunRecord( + run: RunRecordSlot, + change: (record: NativeRunRecord) => void, ): string | undefined { const path = run.path; if (path === undefined) return undefined; const record = JSON.parse(readFileSync(path, "utf8")) as NativeRunRecord; - record.result = "failed"; - if (!record.unhandledErrors.includes(message)) - record.unhandledErrors.push(message); - writeFileSync(`${path}.next`, JSON.stringify(record, null, 2) + "\n"); + change(record); + writeFileSync(`${path}.next`, serializeRunRecord(record, run.secrets)); renameSync(`${path}.next`, path); return path; } diff --git a/tests/pure/evidence.test.ts b/tests/pure/evidence.test.ts index 4a42ee0..97d7404 100644 --- a/tests/pure/evidence.test.ts +++ b/tests/pure/evidence.test.ts @@ -23,6 +23,7 @@ test("pure: simultaneous evidence records differ and an existing record cannot b const directory = await mkdtemp(join(tmpdir(), "libar-evidence-")); const record: NativeRunRecord = { tier: "native", + target: "local backend", commit: "abcdefg", clean: false, command: "vitest run", @@ -32,6 +33,7 @@ test("pure: simultaneous evidence records differ and an existing record cannot b versions: {}, tests: [], unhandledErrors: [], + hosted: null, }; try { vi.mocked(randomUUID) @@ -56,6 +58,7 @@ test("pure: an amendment that fails leaves the old record whole", async () => { const directory = await mkdtemp(join(tmpdir(), "libar-evidence-amend-")); const record: NativeRunRecord = { tier: "native", + target: "local backend", commit: "abcdefg", clean: true, command: "vitest run --project native", @@ -65,6 +68,7 @@ test("pure: an amendment that fails leaves the old record whole", async () => { versions: {}, tests: [], unhandledErrors: [], + hosted: null, }; const actual = await vi.importActual("node:fs"); try { diff --git a/tests/pure/hosted.test.ts b/tests/pure/hosted.test.ts new file mode 100644 index 0000000..4353c7e --- /dev/null +++ b/tests/pure/hosted.test.ts @@ -0,0 +1,175 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, onTestFinished, test } from "vitest"; +import { answerAcceptance } from "../../harness/acceptance.js"; +import type { ScenarioGraph } from "../../harness/acceptance.js"; +import EvidenceReporter, { + amendRunRecord, + hostedDeployMeasurement, + openRunRecord, +} from "../../harness/evidence.js"; +import type { NativeRunRecord } from "../../harness/evidence.js"; +// Binds this file to the harness Spec, whose rules for a native run on a hosted deployment it +// checks with no network and no deploy key. +const anchor = specTest({ + id: testAnchorId("test:platform.native-harness.hosted"), + verifies: ref("spec:platform.native-harness"), +}); +void anchor; + +async function temporary(prefix: string) { + const directory = await mkdtemp(join(tmpdir(), prefix)); + onTestFinished(() => rm(directory, { recursive: true, force: true })); + return directory; +} +const hostedFacts = { + deployment: "happy-animal-123", + keyName: "hosted-ci-scoped", + statedPlan: "Professional", + ranBy: "developer" as const, + cliVersion: "1.46.0", + backendVersion: null, + usageBefore: null, + usageAfter: null, + windowCrossed: false, +}; +// A finished test as the reporter sees it. +function finished( + project: string, + options: { errors?: string[]; measurements?: unknown[] } = {}, +) { + return { + project: { name: project }, + fullName: `${project} test`, + module: { relativeModuleId: `tests/${project}/a.test.ts` }, + result: () => ({ + state: options.errors === undefined ? "passed" : "failed", + errors: (options.errors ?? []).map((message) => ({ message })), + }), + diagnostic: () => ({ duration: 1 }), + meta: () => ({ + native: { backends: [], measurements: options.measurements ?? [] }, + }), + } as never; +} +async function reported( + project: "native" | "hosted", + slot: (run: ReturnType) => void, + options: Parameters[1] = {}, +) { + const directory = await temporary("libar-hosted-record-"); + const reporter = new EvidenceReporter({ directory }); + const run = openRunRecord(); + slot(run); + reporter.onTestRunStart([{ project: { name: project } }] as never); + reporter.onTestCaseResult(finished(project, options)); + await reporter.onTestRunEnd([] as never, [] as never, "passed" as never); + const [name] = await readdir(directory); + const text = await readFile(join(directory, name!), "utf8"); + return { + name: name!, + text, + record: JSON.parse(text) as NativeRunRecord, + run, + }; +} + +test("pure: a record names its target apart from its tier, and a run on a hosted deployment writes hosted- and its hosted facts", async () => { + const local = await reported("native", () => {}); + expect(local.name).toMatch(/^native-/); + expect(local.record).toMatchObject({ + tier: "native", + target: "local backend", + hosted: null, + }); + const deploy = { composition: "fixture", wallMs: 1234 }; + const hosted = await reported("hosted", (run) => (run.hosted = hostedFacts), { + measurements: [{ name: hostedDeployMeasurement, value: deploy }], + }); + expect(hosted.name).toMatch(/^hosted-/); + expect(hosted.record).toMatchObject({ + tier: "native", + target: "hosted deployment", + hosted: { ...hostedFacts, deploys: [deploy] }, + }); + expect(hosted.record.tests[0]!.project).toBe("hosted"); +}); + +test("pure: the reporter and an amendment redact the serialized record by the run's secrets", async () => { + const secret = "dev:happy-animal-123|REVIEW_DUMMY_SECRET"; + const hosted = await reported( + "hosted", + (run) => { + run.hosted = hostedFacts; + run.secrets = [secret]; + }, + { errors: [`failed with ${secret}`] }, + ); + expect(hosted.text).toContain("failed with [redacted]"); + expect(hosted.text).not.toContain(secret); + amendRunRecord(hosted.run, (record) => { + record.unhandledErrors.push(`amended with ${secret}`); + }); + const amended = await readFile(hosted.run.path!, "utf8"); + expect(amended).toContain("amended with [redacted]"); + expect(amended).not.toContain(secret); +}); + +test("pure: the acceptance check reads no record of a native run on a hosted deployment, as the newest or by its path", async () => { + const experiment = "spec:sample.experiment"; + // One required row with no example: the check answers missing, and reads no tier word. + const graph: ScenarioGraph = { + specs: () => [{ id: experiment, specKind: "rule" }], + specContext: (id) => + id === experiment + ? { + sections: { design: { acceptanceRows: "the row is Sc L0-1" } }, + verifiers: [], + } + : undefined, + }; + const directory = await temporary("libar-hosted-acceptance-"); + const record = (startedAt: string, target?: string) => ({ + commit: "0123456789abcdef0123456789abcdef01234567", + clean: true, + result: "passed", + tests: [], + startedAt, + ...(target === undefined ? {} : { target }), + }); + await writeFile( + join(directory, "local.json"), + JSON.stringify(record("2026-01-01T00:00:00.000Z", "local backend")), + ); + await writeFile( + join(directory, "older.json"), + JSON.stringify(record("2025-01-01T00:00:00.000Z")), + ); + await writeFile( + join(directory, "hosted.json"), + JSON.stringify(record("2027-01-01T00:00:00.000Z", "hosted deployment")), + ); + const newest = answerAcceptance({ + graph, + runsDirectory: directory, + specId: experiment, + }); + expect(newest.record).toBe(join(directory, "local.json")); + expect(newest.exit).toBe(3); + const named = answerAcceptance({ + graph, + runsDirectory: directory, + recordPath: join(directory, "hosted.json"), + specId: experiment, + }); + expect(named.exit).toBe(2); + expect(named.lines).toEqual([ + `acceptance: cannot answer · The record ${join(directory, "hosted.json")} is not a run on the local backend, and a native run on a hosted deployment passes no scenario`, + ]); +}); From 0d2edf7b0156f52c631c2cd8fb2d2ed252ce3cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:51:21 +0200 Subject: [PATCH 122/155] design: 4a audit and the diagnostic, the Spec text Co-Authored-By: Claude Opus 5.5 --- .../specs/command/command-declaration.sdp.md | 11 ++--- design/specs/command/command-pipeline.sdp.md | 23 +++++----- ...line-operations.broken-audit-aborts.sdp.md | 15 +++++-- ...erations.broken-metrics-never-abort.sdp.md | 14 +++++-- .../operations/baseline-operations.sdp.md | 42 ++++++++++++------- 5 files changed, 67 insertions(+), 38 deletions(-) diff --git a/design/specs/command/command-declaration.sdp.md b/design/specs/command/command-declaration.sdp.md index 87d00dd..811ee31 100644 --- a/design/specs/command/command-declaration.sdp.md +++ b/design/specs/command/command-declaration.sdp.md @@ -33,7 +33,7 @@ The canonical schema is the declaration's `input` validator plus an optional ref - [non-blocking] Extension E-7: the doc says a small composition helper and shared schema values; the helper signatures, the executor signature and the read-model binding below are the design's; the cost is two registered mutations per command instead of one, and the cheaper alternative the design refuted is one public export per command plus one generic internal dispatcher mutation keyed by command type, as D7's conditional dispatcher already does; it lost because trusted server callers would then reach every command through one untyped `internalMutation`, so a renamed handler or a broken argument contract would pass `tsc` at the caller and fail only at runtime, which is the check Sc L2-4 exists to keep before traffic, and because the receipt fingerprint's contract version would be looked up by string rather than typed; the owner may still prefer the dispatcher form once code generation makes the per-command internal entry free (E-7, D12, D7, Sc L2-4, Decision method rule 5) - [non-blocking] Extension E-7: the pipeline receives the declaration and takes the fingerprint's contract version from it, and the type check at a typed caller is the check of Sc L2-4, so no standing map from command type to declaration exists; the conditional dispatcher of `spec:command.outcome-boundary` is the one thing that would need such a map and brings it with it; this is a provisional reading the owner rules (E-7, D12, D18, OQ2) -- [non-blocking] Extension E-38: the doc names five declaration fields; this design adds `contractVersion` for the fingerprint, `retention` and `irreversible` for receipts, `admission`, `bounds`, `rejections`, `audit` and `writes`, each serving a cited rule of another Spec, and asks whether the owner accepts them as part of the one declaration; `bounds` is the one input bound in the corpus, `maxItems` and `maxBytes`, checked by the pipeline's step 1 and by no executor body, so `spec:application.parent-use-cases` reads it rather than declaring a bound of its own; `writes` is required on every declaration, because an absent list would be an unchecked one, and `readModels` is optional, absent meaning none, which the owner has not ruled; `writes` names the context and stream type pairs the executor may write, from which the pipeline's step 7 derives the source scopes the write pause closes for a consistent cut and which step 9 proves against the `streams` entries the executor returns, because a use case that writes a source stream without maintaining the paused view would otherwise pass the gate; the declaration still imports nothing from the obligation family, so a command's static reactions are not a field here but the `createObligation` calls of its executor, as `spec:obligations.fan-out-and-chains` rules under E-54 (E-38, E-8, E-54, D6, D9, D10, D19, Thesis) +- [non-blocking] Extension E-38: the doc names five declaration fields; this design adds `contractVersion` for the fingerprint, `retention` and `irreversible` for receipts, `admission`, `bounds`, `rejections`, `audit`, `diagnosticSink` and `writes`, each serving a cited rule of another Spec, and asks whether the owner accepts them as part of the one declaration; `bounds` is the one input bound in the corpus, `maxItems` and `maxBytes`, checked by the pipeline's step 1 and by no executor body, so `spec:application.parent-use-cases` reads it rather than declaring a bound of its own; `writes` is required on every declaration, because an absent list would be an unchecked one, and `readModels` is optional, absent meaning none, which the owner has not ruled; `writes` names the context and stream type pairs the executor may write, from which the pipeline's step 7 derives the source scopes the write pause closes for a consistent cut and which step 9 proves against the `streams` entries the executor returns, because a use case that writes a source stream without maintaining the paused view would otherwise pass the gate; the declaration still imports nothing from the obligation family, so a command's static reactions are not a field here but the `createObligation` calls of its executor, as `spec:obligations.fan-out-and-chains` rules under E-54 (E-38, E-8, E-54, D6, D9, D10, D19, Thesis) ## Contract @@ -48,23 +48,24 @@ The canonical schema is the declaration's `input` validator plus an optional ref - No DI container, decorator framework, reflection registry or module lifecycle; a class may group a module's declarations and holds no correctness-critical global state (D12) - Generation of the repetitive bindings starts only after two real modules show the pattern, is deterministic, and CI fails when regenerating changes a committed file (D12) - Renaming a handler export changes the generated `api` type, so every typed caller fails `tsc`; breaking an argument contract fails `tsc` at the caller and the `args` validator at runtime (D12, Sc L2-4) -- [extension] The declaration also carries `contractVersion`, `retention`, `irreversible`, `admission`, `bounds`, `rejections`, `audit` and `writes`, each read by one cited rule elsewhere in the corpus; `bounds` is checked once, at the pipeline's step 1, and `writes` is proven once, at its step 9 (E-38, E-8) +- [extension] The declaration also carries `contractVersion`, `retention`, `irreversible`, `admission`, `bounds`, `rejections`, `audit`, `diagnosticSink` and `writes`, each read by one cited rule elsewhere in the corpus; `bounds` is checked once, at the pipeline's step 1, and `writes` is proven once, at its step 9 (E-38, E-8) - [extension] The declaration carries no reactions field, because a lower profile never imports a higher profile's machinery; the static reactions of a command in the durable profile are the `createObligation` calls its executor makes, bounded and reviewed as `spec:obligations.fan-out-and-chains` rules (E-38, E-54, Thesis, D13) ## Design -The helpers are ordinary functions. `publicCommand` builds the public entry's `args` from `tenantId`, an optional `requestKey`, an optional `correlationId` and the declaration's `input`, sets `returns` to the pipeline's `commandResponseValidator(decl.output)`, and writes a handler that establishes the actor, which happens after Convex's argument validators and before step 1's key, bound and refinement checks, fixes the namespace to `public` and calls `runPipeline`. `internalCommand` builds the internal entry's `args` with `namespace`, `actor`, a required `requestKey`, the same optional `correlationId` and an optional `causedBy`, and calls the same `runPipeline`. Both return the `RegisteredMutation` Convex expects at a top-level export, and both forward `correlationId` on the `PipelineCall` so that step 7 stores it on the `OperationRef`; only the internal entry forwards `causedBy`, which step 7 stores the same way. Each handler's one outermost catch is `normalizeThrown(error, decl.name, decl.rejections)` of `spec:command.outcome-boundary`. +The helpers are ordinary functions. `publicCommand` builds the public entry's `args` from `tenantId`, an optional `requestKey`, an optional `correlationId` and the declaration's `input`, sets `returns` to the pipeline's `commandResponseValidator(decl.output)`, and writes a handler that establishes the actor, which happens after Convex's argument validators and before step 1's key, bound and refinement checks, fixes the namespace to `public` and calls `runPipeline`. `internalCommand` builds the internal entry's `args` with `namespace`, `actor`, a required `requestKey`, the same optional `correlationId` and an optional `causedBy`, and calls the same `runPipeline`. Both return the `RegisteredMutation` Convex expects at a top-level export, and both forward `correlationId` on the `PipelineCall` so that step 7 stores it on the `OperationRef`; only the internal entry forwards `causedBy`, which step 7 stores the same way. Each handler's one outermost catch is `relayFailure` of `spec:command.command-pipeline`, which emits the failure's diagnostic record and throws what `normalizeThrown(error, decl.name, decl.rejections)` of `spec:command.outcome-boundary` throws. What generation would emit, once two modules show the pattern: for each declaration in a module, the two export lines and a module-level union of rejection codes for the outcome boundary. It emits nothing that a hand-written module could not contain, and CI regenerates and diffs. -- typeCommandDeclaration: `type CommandDeclaration = { name: string; contractVersion: number; input: Validator; refine?: (input: I) => Omit | null; output: Validator; permission: PermissionPolicy; executor: Executor; writes: readonly SourceRef[]; readModels?: readonly ReadModelBinding[]; rejections: readonly string[]; admission?: AdmissionPolicy; bounds?: Bounds; retention?: Retention; irreversible?: boolean; audit?: { kind: "security" | "business" } }` (D12, E-7, E-38) +- typeCommandDeclaration: `type CommandDeclaration = { name: string; contractVersion: number; input: Validator; refine?: (input: I) => Omit | null; output: Validator; permission: PermissionPolicy; executor: Executor; writes: readonly SourceRef[]; readModels?: readonly ReadModelBinding[]; rejections: readonly string[]; admission?: AdmissionPolicy; bounds?: Bounds; retention?: Retention; irreversible?: boolean; audit?: { kind: "security" | "business" }; diagnosticSink?: DiagnosticSink }` (D12, E-7, E-38) - typeSourceRef: `type SourceRef = { contextId: string; streamType: string }`, one context and stream type pair, used by `writes` and by `ReadModelBinding.source` (E-38, E-8) - typeWrites: `writes` lists every `SourceRef` the executor's context calls may write; the pipeline's step 7 turns each into a `source:` scope for the write pause's gate read, and its step 9 throws a plain error when a returned `streams` entry names a pair outside the list, so an understated list fails the first native run instead of slipping through a closed source scope; a use case that writes nothing outside its declared pairs is the only kind the gate can cut consistently (E-38, E-8, D9, Law 6) - typePermissionPolicy: `type PermissionPolicy = { permission: string; subjectFrom?: (input: I) => SubjectRef }` (D11, D12) - typeExecutor: `type Executor = (ctx: MutationCtx, call: { tenantId: string; actor: Actor; operation: OperationRef; input: I }) => Promise>` where `OperationRef` is the one the pipeline minted at step 7, forwarded unchanged to every context call, and the executor's context calls are `ctx.runMutation` on component APIs with the facts it captured (D12, D10, D4, D3, E-26) - typeExecutorResult: `type ExecutorResult = CommittedOutcome & { streams: StreamDto[] }` where `CommittedOutcome` is the kernel's committed half of `Outcome` and `streams` concatenates the `streams` of every context outcome the executor received, the `StreamDto` of `spec:context.persistence-adapter`, so the pipeline's step 9 has every DTO, version and created flag it needs (D12, D4, D8, E-7, E-23) - typeReadModelBinding: `type ReadModelBinding = { readModel: ReadModel; source: SourceRef }`, where `readModel` is the declaration `spec:application.projection-contract` pins and `source` selects which `streams` entries feed it; the binding repeats neither the read model's name nor its projection's version, a read model fed by several stream types is bound once per source, and the binding has no update function of its own (D8, D12, E-7, E-9) -- typeAudit: `audit?: { kind: "security" | "business" }` asks the pipeline's step 10 to write one audit record through `writeAudit` of `spec:operations.baseline-operations`, which fails closed (D19, E-38, E-43) +- typeAudit: `audit?: { kind: "security" | "business" }` asks the pipeline's step 10 to write one audit record through `writeAudit` of `spec:operations.baseline-operations`, which fails closed, after an applied command and after a business failure; absent, the default, the command writes none (D19, E-38, E-43) +- typeDiagnosticSinkField: `diagnosticSink?: DiagnosticSink`, the sink type of `spec:operations.baseline-operations`, names where the pipeline writes the command's diagnostic lines; absent, the default, it is `consoleSink`; the production composition names none, and the fixture composition's `PlaceOrder` names the sink through which Sc ALL-1 breaks metrics and logging (D19, E-38, E-43) - typeAdmissionPolicy: `type AdmissionPolicy = (ctx: MutationCtx, call: PipelineCall) => Promise<{ admitted: true } | { admitted: false; code: "rateLimited" | "capacity"; retryAfterMs?: number }>`; a consumption it writes commits only with the command, so it counts committed intent (D6, E-32) - typeBounds: `type Bounds = { maxItems?: number; maxBytes?: number }` checked by the pipeline at step 1 and nowhere else, where `maxItems` counts the elements of every top-level array field of the input, summed, and `maxBytes` bounds the input's size measured with `getConvexSize`; a breach is rejection `operationTooLarge` carrying the bound and the offered size, and `maxItems` is the number the first experiment measures under as the use case's input bound (D10, F13, E-38) - typeRetention: `type Retention = { window: number; afterExpiry: "delete" | "tombstone" }` (D6, E-34) diff --git a/design/specs/command/command-pipeline.sdp.md b/design/specs/command/command-pipeline.sdp.md index a499d59..c9ea28e 100644 --- a/design/specs/command/command-pipeline.sdp.md +++ b/design/specs/command/command-pipeline.sdp.md @@ -67,7 +67,7 @@ The pipeline is one code path with two registered entries per command. The publi ## Workflow - rule: A business operation runs as one top-level mutation; the steps below never split across transactions (D1) -- rule: Failure handling inside the mutation is a throw; the pipeline never catches to continue, its one catch at the handler's outermost level only rethrows in the boundary's wire shape, and it never converts a technical failure into a rejection (D1, D7, Law 6) +- rule: Failure handling inside the mutation is a throw; the pipeline never catches to continue, its one catch at the handler's outermost level only rethrows in the boundary's wire shape, after emitting the failure's diagnostic record through `emitDiagnostic`, which cannot throw, and it never converts a technical failure into a rejection (D1, D7, Law 6, D19) - rule: Authorization precedes execution and precedes the disclosure of any stored outcome (Law 5, D11) - rule: The receipt lookup precedes admission, so a valid retry is never refused for capacity (D6, Sc L1-9) - rule: Only the parent reads `ctx.auth`; every context receives the server-established actor and tenant scope as arguments (D11, F11) @@ -81,21 +81,21 @@ The pipeline is one code path with two registered entries per command. The publi - Step 2, authenticate: the public entry calls `establishActor` of `spec:command.actor-and-scope`, which reads `ctx.auth.getUserIdentity()` once, after Convex's argument validators and before step 1's key, bound and refinement checks, and throws rejection `unauthenticated` on `null`; the internal entry accepts the `actor` argument its trusted caller established and performs no authentication of its own (D11, F11) - Step 3, establish scope: the `tenantId` argument is the tenant scope for every read and write that follows; the pipeline resolves the actor's membership in that tenant through the grants read of step 4 and never infers a tenant from the identity (Law 11, D11) - Step 4, authorize: the pipeline reads the actor's grants for the tenant inside the transaction and evaluates the declaration's permission policy against actor, tenant scope and input; a denial throws rejection `forbidden` with `details` `{ reason }`, the reason `authorize` gave, before any execution and before any receipt is read (Law 5, D11, D12) -- Step 5, receipt lookup: for a receipted call the pipeline builds the key from tenant, namespace, command type and request key and reads the `receipts` table by its key index in the same transaction; no receipt means new intent, a receipt whose `contractVersion` differs from the declaration's throws rejection `unsupportedContractVersion` before any fingerprint is compared, a receipt with the same fingerprint means a duplicate and the pipeline returns the stored thin outcome with `replayed` true and `result` null, a receipt with a different fingerprint throws rejection `idempotencyConflict`, and neither rejection carries details or anything else of the stored receipt except the one fact the two codes cannot hide, whether the receipt's contract version is the declaration's (D6, Law 4, Law 5, Sc L1-3, Sc L1-5, E-5) +- Step 5, receipt lookup: for a receipted call the pipeline builds the key from tenant, namespace, command type and request key and reads the `receipts` table by its key index in the same transaction; no receipt means new intent, a receipt whose `contractVersion` differs from the declaration's throws rejection `unsupportedContractVersion` before any fingerprint is compared, a receipt with the same fingerprint means a duplicate and the pipeline emits the duplicate's diagnostic record and returns the stored thin outcome with `replayed` true and `result` null, a receipt with a different fingerprint throws rejection `idempotencyConflict`, and neither rejection carries details or anything else of the stored receipt except the one fact the two codes cannot hide, whether the receipt's contract version is the declaration's (D6, Law 4, Law 5, Sc L1-3, Sc L1-5, E-5) - Step 5, disclosure: a duplicate's stored outcome is returned only because step 4 ran for this call, so a caller whose authorization was revoked after the original run receives `forbidden` and learns nothing (D6, Law 5, Sc L1-8) - Step 6, admit: for new intent only, the pipeline evaluates the declaration's admission policy; a refusal throws transient `rateLimited` or `capacity`, nothing is stored, and the same intent may be retried (D6, D4, Sc L1-9) - Step 7, gate and mint the operation: the pipeline reads the maintenance gate once through `assertWritable(ctx, scopesOfUseCase(tenantId, decl.writes))` of `spec:application.write-pause`, which refuses with the transient `writePaused` while a scope of this command is closed, the whole deployment, the tenant or a stream type it writes, or the deployment is in restore, and only then mints the whole `OperationRef`, the operation ID, `causedBy` from the call's `causedBy` argument when the internal entry received one and otherwise of kind `command` with the declaration's name, and the caller's optional correlation ID, and passes it to every context call so that events in different streams relate through it (Vocab: operation, D2, D9, D13, E-8, E-26, E-31) - Step 8, execute: the executor captures the outside facts the decision needs, from parent tables or configuration read in this transaction, and calls each context named by the declaration once through `ctx.runMutation` on the component's sanctioned operation, `components..operations.`, with tenant scope, actor, the operation reference, the command's input as lists and the captured facts; each call is an isolated sub-transaction that commits with the parent and rolls back on throw; a rejection thrown inside a context rolls its sub-transaction back and propagates to the boundary's outermost rethrow, and the executor returns the kernel's `CommittedOutcome` together with the `streams` entries every context call returned, never a rejection value (F2, F3, D3, D10, D7, E-23) -- Step 9, update read models: the pipeline first checks that every `streams` entry the executor returned names a context and stream type in the declaration's `writes`, throwing a plain error, a defect and a technical failure, when one does not, so the list the gate of step 7 was derived from is proven on the first native run; then, once for each read model its bindings name, however many bindings name it, it reads the generations to write through `generationsToWrite` and throws a plain error that names the command and the read model when there is none, and, for every binding, calls `applyProjection` of `spec:application.projection-contract` for each matching entry with the entry's DTO and version and the list unchanged, in the live-created mode when the entry's `created` flag is set and the live-updated mode otherwise, or `applyAggregate` or `applyHistoryProjection` for a binding of those forms, in the same transaction; it counts the rows those calls insert, replace or delete, a row in each generation once, and after each entry is applied throws a plain error when they pass `limitReadModelWritesPerCommand`; the executor writes no read-model table itself (D8, D9, D12, Law 9, E-7, E-8, E-9, E-23, E-45) -- Step 10, insert the receipt and the audit record: for a receipted call the pipeline inserts the thin receipt, outcome kind, operation ID, affected IDs derived from the stream versions, and the stream versions, in the same transaction as the lookup of step 5, so two concurrent identical requests serialize under optimistic concurrency and only one effect commits; for a declaration that sets `audit` it then calls `writeAudit` of `spec:operations.baseline-operations`, whose failure throws and rolls the command back (D6, D19, F1, Sc L1-3, E-43) -- Step 11, return: the mutation emits one diagnostic record through `emitDiagnostic` inside a try and catch that swallows every error, then returns the outcome kind, the result DTO, the operation ID, the affected IDs, the stream versions and `replayed` false (D4, D8, D2, D19, E-43) +- Step 9, update read models: the pipeline first checks that every `streams` entry the executor returned names a context and stream type in the declaration's `writes`, throwing a plain error, a defect and a technical failure, when one does not, so the list the gate of step 7 was derived from is proven on the first native run; then, once for each read model its bindings name, however many bindings name it, it reads the generations to write through `generationsToWrite` and throws a plain error that names the command and the read model when there is none, and, for every binding, calls `applyProjection` of `spec:application.projection-contract` for each matching entry with the entry's DTO and version and the list unchanged, in the live-created mode when the entry's `created` flag is set and the live-updated mode otherwise, or `applyAggregate` or `applyHistoryProjection` for a binding of those forms, in the same transaction; it counts the rows those calls insert, replace or delete, a row in each generation once, and after each entry is applied throws a plain error when they pass `limitReadModelWritesPerCommand`, and the count is what the diagnostic record carries as `readModelRows`; the executor writes no read-model table itself (D8, D9, D12, Law 9, E-7, E-8, E-9, E-23, E-45) +- Step 10, insert the receipt and the audit record: for a receipted call the pipeline inserts the thin receipt, outcome kind, operation ID, affected IDs derived from the stream versions, and the stream versions, in the same transaction as the lookup of step 5, so two concurrent identical requests serialize under optimistic concurrency and only one effect commits; for a declaration that sets `audit` it then calls `writeAudit` of `spec:operations.baseline-operations` with the record `auditRecordSources` of that Spec builds, whose failure throws and rolls the command back, a refused insert being a plain error and so a technical failure with nothing of the command stored (D6, D19, F1, Sc L1-3, E-43) +- Step 11, return: the mutation builds the diagnostic record of the outcome from values it already holds and passes it to `emitDiagnostic` with the declaration's sink, which swallows every error, then returns the outcome kind, the result DTO, the operation ID, the affected IDs, the stream versions and `replayed` false (D4, D8, D2, D19, E-43) - At any step, a throw leaves the mutation uncommitted: the state and journal writes of every context sub-transaction, the read-model writes and the receipt are all rolled back, and nothing is stored as a business outcome (Law 2, D7, F2, Sc L1-2) ## Design -The transaction boundary is the top-level mutation. Inside it, the parent's own code runs as plain helpers with no nested `ctx.runMutation`, because a helper's writes roll back with the mutation and a nested call costs more than a helper. The only sub-transactions are the component calls of step 8, one per context, because a component can only be reached through `ctx.runMutation` or `ctx.runQuery` and each such call is isolated from the others and commits with the caller. The pipeline never catches a component's throw to continue; its outermost `normalizeThrown` only rethrows, so that a context's bare kernel `Rejection` and the pipeline's own rejections share one wire shape, and a context's code that is neither a platform code nor in the declaration's `rejections` becomes a technical failure. The nested-mutation form that D7 keeps for a worker wrapper or a refusal record lives in the obligation module and in the conditional dispatcher of `spec:command.outcome-boundary`, not here. +The transaction boundary is the top-level mutation. Inside it, the parent's own code runs as plain helpers with no nested `ctx.runMutation`, because a helper's writes roll back with the mutation and a nested call costs more than a helper. The only sub-transactions are the component calls of step 8, one per context, because a component can only be reached through `ctx.runMutation` or `ctx.runQuery` and each such call is isolated from the others and commits with the caller. The pipeline never catches a component's throw to continue; its outermost catch only rethrows, through `relayFailure` and `normalizeThrown`, after emitting the failure's diagnostic record, so that a context's bare kernel `Rejection` and the pipeline's own rejections share one wire shape, and a context's code that is neither a platform code nor in the declaration's `rejections` becomes a technical failure. The nested-mutation form that D7 keeps for a worker wrapper or a refusal record lives in the obligation module and in the conditional dispatcher of `spec:command.outcome-boundary`, not here. -The pipeline depends on three contracts it does not own: the context component's operation mutations, one per sanctioned operation, and their returned DTO, versions and `streams`, the receipt table and its key index, and the actor, scope and grants shapes. It cites Package B's operations at the point the step sequence calls them and states nothing about the adapter's inner steps. Four steps call helpers that Layer 2 Specs pin, the gate of step 7, the projections and the generation registry of step 9, and the audit and diagnostic of steps 10 and 11; each is a no-op in a deployment that has no read model, no closed gate and no audited declaration, so a Layer 1 application runs the pipeline before Layer 2 exists, and the graph edges run from those Specs toward this one, never the reverse. +The pipeline depends on three contracts it does not own: the context component's operation mutations, one per sanctioned operation, and their returned DTO, versions and `streams`, the receipt table and its key index, and the actor, scope and grants shapes. It cites Package B's operations at the point the step sequence calls them and states nothing about the adapter's inner steps. Four steps call helpers that Layer 2 Specs pin, the gate of step 7, the projections and the generation registry of step 9, and the audit and diagnostic of steps 10 and 11; each but the diagnostic, which is one log line, is a no-op in a deployment that has no read model, no closed gate and no audited declaration, so a Layer 1 application runs the pipeline before Layer 2 exists, and the graph edges run from those Specs toward this one, never the reverse. - transactionBoundary: one top-level mutation per command; each context call is a component sub-transaction through `ctx.runMutation`; no nested `runMutation` on the parent's own code on the happy path (D1, F2, F3) - convexSurface: per command, one public `mutation` export for Convex clients and one `internalMutation` export for server-side callers, both from the composition helper; the parent registers no other function per command, and the conditional dispatcher of `spec:command.outcome-boundary` runs the internal entry as its nested body rather than a body of its own (D12, D6, D7) @@ -116,10 +116,11 @@ The pipeline depends on three contracts it does not own: the context component's - rejectionRelay: a `ConvexError` carrying the kernel's `Rejection` thrown inside a component reaches the pipeline with its data intact under F14, which Probe 2 showed on the pinned backend; `normalizeThrown(error, decl.name, decl.rejections)` at the handler's outermost level rethrows it as the boundary's `RejectionData` when its code is a platform code or one the declaration's `rejections` names, and as a plain error otherwise; the fallback for a refuted F14, contexts returning the rejection as a value, is not needed (F14, Probe 2, D7) - readModelWrites: parent-owned read-model tables written by the pipeline at step 9, one `applyProjection`, `applyAggregate` or `applyHistoryProjection` call per matching `streams` entry per binding the declaration lists, with the list `generationsToWrite` returns, each generation with its role; the executor holds the DTOs and the appended events only long enough to return them and writes no read-model table, so the declaration's list is the one wiring fact for read models (D8, D9, D12, E-7, E-9) - writesCheck: `decl.writes` is proven at step 9 against the `streams` entries the executor returned, in memory and with no extra read; an entry outside the list throws a plain error, so a declaration whose `writes` understates its executor fails its first native run rather than passing a closed source scope at step 7 (E-8, E-38, Law 6) -- fnWriteReadModels: `writeReadModels(ctx: RegistryReader & RowWriter, commandType: string, bindings: readonly ReadModelBinding[], input: { tenantId: string; streams: readonly StreamDto[] }): Promise`, the read-model half of step 9 and the one function of the read-model library the pipeline calls, returning the number of entries it fed; the pipeline passes its own ctx, which at run time is the parent's and holds the registry and the read model's table (D8, E-7, E-9) +- fnWriteReadModels: `writeReadModels(ctx: RegistryReader & RowWriter, commandType: string, bindings: readonly ReadModelBinding[], input: { tenantId: string; streams: readonly StreamDto[] }): Promise`, the read-model half of step 9 and the one function of the read-model library the pipeline calls, returning the number of read-model rows it inserted, replaced or deleted, a row in each generation once, the count it holds against `limitReadModelWritesPerCommand`; the pipeline passes its own ctx, which at run time is the parent's and holds the registry and the read model's table (D8, E-7, E-9) - helperGenerics: [extension] every library helper that takes the parent's own ctx or reads one of the parent's tables, `writeReadModels`, `activeGeneration` and `authorizeQuery` among them, is generic over the parent's data model, because a database reader is not covariant in its data model under the repository's compiler settings, so a helper typed with the library's tables alone refuses the parent's ctx (E-7, E-9, D11) -- auditWrite: `writeAudit` of `spec:operations.baseline-operations` at step 10 for a declaration whose `audit` field is set, inside the transaction and after the receipt insert; its failure throws and the command rolls back (D19, E-43, Sc ALL-1) -- diagnosticEmit: `emitDiagnostic` of `spec:operations.baseline-operations` at step 11, after every business write and inside a try and catch that swallows every error, so no diagnostic failure reaches the mutation (D19, E-43, Sc ALL-1) +- auditWrite: `writeAudit` of `spec:operations.baseline-operations` at step 10 for a declaration whose `audit` field is set, inside the transaction and after the receipt insert, given the `AuditRecordInput` that `auditRecordSources` of that Spec builds; its failure throws, the outermost catch passes the plain error through unchanged, the caller sees a technical failure and the command rolls back with nothing stored; a duplicate returns at step 5 before it and a throw at any earlier step skips it (D19, E-43, Sc ALL-1) +- diagnosticEmit: `emitDiagnostic` of `spec:operations.baseline-operations` once per execution of an entry's handler, with the declaration's `diagnosticSink` or else `consoleSink`: at step 11 after every business write, at step 5 just before a duplicate is returned, and in the outermost catch through `relayFailure`; the record's fields come from the sources that `diagnosticSourcesReturned`, `diagnosticSourcesDuplicate` and `diagnosticSourcesThrown` of that Spec list, and `emitDiagnostic` swallows every error, so no diagnostic failure reaches the mutation (D19, E-43, Sc ALL-1) +- fnRelayFailure: `relayFailure(decl: CommandDeclaration, call: FailedCall, error: unknown): never` in `src/command/pipeline.ts` is the whole body of each entry's outermost catch: it captures what `normalizeThrown(error, decl.name, decl.rejections)` of `spec:command.outcome-boundary` throws, emits the record of that throw, and throws it unchanged, so the caller receives exactly what `normalizeThrown` alone would throw; `type FailedCall = { tenantId: string; namespace: CallerNamespace; actor?: Actor; requestKey?: string; correlationId?: string; causedBy?: CausedBy }` holds what the entry has when it throws, `actor` absent while the public entry has established none (D7, D19, E-43) - receiptWrites: the `receipts` table of `spec:command.receipt-table`, read at step 5 by `by_key` and inserted at step 10 in the same mutation (D6) - admissionHook: `admission?: (ctx: MutationCtx, call: PipelineCall) => Promise<{ admitted: true } | { admitted: false; code: "rateLimited" | "capacity"; retryAfterMs?: number }>` on the declaration, absent by default; a refusal throws the transient error and any consumption it wrote rolls back with the throw, as it does with every later throw of the same mutation, so an admitted call that is then rejected leaves no consumption either (D6, D4, E-32) - subTransactionLimits: `ctx.runMutation` accepts `{ transactionLimits }` per call; the limits are shared across the boundary, as Probe 4 showed; the design passes no `transactionLimits` on a context call, and whether a call should reserve room for the work after it is an open question (S3, Probe 4, F13) @@ -146,7 +147,7 @@ And an event from the command exists in the journal {eventExists:boolean} ## Verification — reviewed -- A reviewer confirms that the eleven steps run inside one registered mutation, that the only `ctx.runMutation` calls are the context calls of step 8, and that the only catches are the handler's outermost rethrow, which never continues, and the try around `emitDiagnostic`, which swallows and writes nothing. +- A reviewer confirms that the eleven steps run inside one registered mutation, that the only `ctx.runMutation` calls are the context calls of step 8, and that the only catches are the handler's outermost catch, which hands the throw to `relayFailure` and never continues, `relayFailure`'s capture of what `normalizeThrown` throws, which it throws again unchanged, and the two inside `emitDiagnostic`, which swallow and write nothing to the database. - A reviewer confirms that the gate read of step 7 precedes the first context call, that step 9 is the only live writer of read-model tables and takes its DTOs and events from the executor's `streams`, that step 9 checks every entry against `decl.writes`, and that the executor passes the minted `OperationRef` and its captured `facts` to every context call unchanged. - A reviewer confirms that the internal entry's `causedBy` argument is the only way an `OperationRef` gets a cause of kind `event`, and that the public entry has no such argument. - A reviewer confirms that the receipt lookup of step 5 precedes the admission of step 6 and that the authorization of step 4 precedes both execution and the disclosure of a duplicate's outcome. diff --git a/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md b/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md index e16b2b2..371c3f1 100644 --- a/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md +++ b/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md @@ -9,7 +9,9 @@ relations: --- # Break mandatory audit -Sc ALL-1 · native tier · the second of the two enumerated cases; applies to every installed layer. +Sc ALL-1 · native tier · fixture composition · the second of the two enumerated cases; applies to every installed layer. + +The fault sits in the fixture composition's `PlaceOrder`, whose declaration sets `audit` and whose `subjectFrom` gives the order ID `auditFaultOrderId` a subject with one field the `auditRecords` validator does not declare. An ordinary client with grants sends the receipted `PlaceOrder` for that order. The depot creates the order and claims its stock, step 10 inserts the receipt, and Convex refuses the audit record's insert, so the plain error ends the mutation and nothing of the command survives. The fault reads no switch and touches no shared library: authorization compares only the subject's three fields, and every other order ID gets the plain subject. ## Intent @@ -25,6 +27,11 @@ And the failure is {surfaced: "returned to the caller as a technical failure"} ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. -- The test points the fixture app's audit writer at a table whose validator refuses the record and runs `PlaceOrder`. -- The test asserts that the caller sees a technical failure, not a rejection code, and that no order, event, receipt, summary row or audit record exists for the operation. +- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- The verifier is `tests/native/baseline-operations.broken-audit-aborts.test.ts`, whose `specTest` anchor `test:operations.baseline-operations.broken-audit-aborts` verifies this example. +- The first Given step starts `fixtureBackend()`, grants the fixture's `orderPermission` and `permissions.stock` to the human principal `${issuer}|user-1` in the tenants `tenant-all-1` and `sinkFaultTenant` through `grants:grant` with admin access, and, as that caller through an ordinary client carrying the fixture issuer's token for `user-1`, adds 10 units of the product `p-1` in each tenant through `depotCommands:addStock`. +- The subsystem step for mandatory audit selects the tenant `tenant-all-1` and the order ID `auditFaultOrderId`. +- The When step reads the depot's `events` and `streams` and the parent's `receipts` and `auditRecords` with admin access, takes a log mark, calls `orders:placeOrder` through the ordinary client with the selected tenant, the request key `all-1-key` and the input `{ orderId, title: "Order", lines: [{ productId: "p-1", quantity: 1 }] }`, keeps the answer or the error, and reads the completion records after the mark until one has the identifier `orders:placeOrder`. +- The Then step for a command that rolls back asserts that the four tables hold exactly what they held before the call. +- The step for a technical failure asserts that the error is not a `ConvexError`, so `classifyThrown` reads it as technical, that the completion record's `error` contains `auditFault`, that exactly one of its `logLines` contains `diagnostic {` and that line contains `technical`, `tenant-all-1` and `all-1-key` and does not contain `operationId`, and that none of its `logLines` contains `diagnostic gap`. +- The test then shows on the same deployment that the audit writer runs: `orders:placeOrder` in `tenant-all-1` with the order ID `order-control` and the request key `all-1-control` returns `applied`, and `auditRecords` holds exactly one record, whose `operationId` is the one returned and whose `decision` is `applied`. diff --git a/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md b/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md index b666611..01240bd 100644 --- a/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md +++ b/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md @@ -9,7 +9,9 @@ relations: --- # Break metrics and logging -Sc ALL-1 · native tier · the first of the two enumerated cases; applies to every installed layer. +Sc ALL-1 · native tier · fixture composition · the first of the two enumerated cases; applies to every installed layer. + +The fault sits in the fixture composition's diagnostic sink: `PlaceOrder` names `fixtureDiagnosticSink`, which throws for every line of the tenant `sinkFaultTenant`. An ordinary client with grants sends the receipted `PlaceOrder` in that tenant. The command commits its events, its receipt and its audit record; at step 11 the sink throws, `emitDiagnostic` swallows the error and writes one gap line in place of the lost record, and the client receives the applied answer. ## Intent @@ -25,6 +27,10 @@ And the failure is {surfaced: "reported as a diagnostic gap without touching the ## Verification — executable -- Runs in the native tier; every test owns its disposable backend. -- The test replaces the fixture app's diagnostic sink with one that throws on every call and runs `PlaceOrder`. -- The test asserts that the command returned applied, that the order, its events, its receipt, its summary row and its audit record exist, and that the fixture's swallowed-error counter is one. +- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- The verifier is `tests/native/baseline-operations.broken-metrics-never-abort.test.ts`, whose `specTest` anchor `test:operations.baseline-operations.broken-metrics-never-abort` verifies this example. +- The first Given step and the When step are those of `spec:operations.baseline-operations.broken-audit-aborts`. +- The subsystem step for metrics and logging selects the tenant `sinkFaultTenant` and the order ID `order-all-1`. +- The Then step for a command that commits asserts that the answer has `kind` `applied` and `replayed` false; that the depot's `events` hold exactly two events whose `operationId` is the one returned, one on the `document` stream `order-all-1` and one on the `stock` stream `p-1`; that `receipts` holds exactly one receipt, with the request key `all-1-key`, that operation ID and the outcome `applied`; and that `auditRecords` holds exactly one record, with the tenant `tenant-sink-fault`, that operation ID, the request key `all-1-key`, the `commandType` `PlaceOrder`, the `kind` `business`, the `decision` `applied` and the subject `{ contextId: "depot", streamType: "document", streamId: "order-all-1" }`. +- The step for a diagnostic gap asserts that the completion record's `error` is null, that exactly one of its `logLines` contains `diagnostic gap` and that line contains the operation ID returned and `applied`, and that none of its `logLines` contains `diagnostic {`; that one gap line is the count of one swallowed error. +- The test then shows on the same deployment that the sink runs: `orders:placeOrder` in `tenant-all-1` with the order ID `order-control` and the request key `all-1-control` returns `applied`, and its completion record holds exactly one line that contains `diagnostic {`, a line that also contains its operation ID and `applied`, and no line that contains `diagnostic gap`. diff --git a/design/specs/operations/baseline-operations.sdp.md b/design/specs/operations/baseline-operations.sdp.md index 390bc28..7b913c7 100644 --- a/design/specs/operations/baseline-operations.sdp.md +++ b/design/specs/operations/baseline-operations.sdp.md @@ -33,20 +33,20 @@ Security, bounded resource use, useful errors and tests start in Layer 1, so non - problem: Metrics and logging break and a valid write is lost with them; separately, mandatory audit breaks and a security-sensitive command commits without its record; a batch runs at the platform limit and times out halfway; a deletion request meets events that hold personal data inline and nobody can say which claims survive (Sc ALL-1, D19) - outcome: Diagnostics never abort valid work and an audit failure does; every batch has a stated bound; events stay small with personal data behind references; a retention and deletion policy exists before production data; diagnosis works by identifiers with stored receipts as the authority; the metrics the layers create are measured; and a release preserves, migrates or drains pending work (D19, Law 8) - value: A maintainer traces one command's success or failure by its identifiers without unrelated machinery, and an operator knows a batch's bound before it runs (D19, Thesis) -- risk: The standing cost is an audit record per audited command, a diagnostic record per command, a metric list per installed layer, a retention policy and a recovery plan (D19) +- risk: The standing cost is an audit record per audited command, a diagnostic line per execution of a command entry, a metric list per installed layer, a retention policy and a recovery plan (D19) - assumption: Backups exclude pending scheduled functions, environment variables, code and configuration (F12) - assumption: Transactions have limits, and the mutation timeout is one second (F13) ### Open questions -- [non-blocking] Extension E-43: the doc says audit is written inside the transaction and fails closed, that diagnostics hold no raw prompts, credentials or unbounded payloads, and that personal data sits behind references; it does not give the audit record, the diagnostic record or the reference. The option taken here: an `auditRecords` table with indexes by operation, subject and request key, whose every field has a stated source in the command, an `operatorAudit` table for the gate changes an operator entry makes, which name no tenant when the scope is the whole deployment, a bounded diagnostic record emitted through a sink that never throws, and a `personalData` table that events point at by reference. The do-nothing option is Convex's own audit logging, which the pinned `convex` package declares as `audit(body)`; it is not taken as the record, because an entry in a log outside the database is not read by the operator's queries by operation, subject and request key, and whether such an entry proves a commit is not shown by the pinned package; a deployment may add it beside the tables as an export (E-43, D19) +- [non-blocking] Extension E-43: the doc says audit is written inside the transaction and fails closed, that diagnostics hold no raw prompts, credentials or unbounded payloads, and that personal data sits behind references; it does not give the audit record, the diagnostic record or the reference. The option taken here: an `auditRecords` table with indexes by operation, subject and request key, whose every field has a stated source in the command, an `operatorAudit` table for the gate changes an operator entry makes, which name no tenant when the scope is the whole deployment, a bounded diagnostic line for every execution of a command entry's handler, whatever it ends in, written through a sink the declaration may name by a function that never throws, with no duration, because the backend records the time of each execution, and a `personalData` table that events point at by reference. The do-nothing option is Convex's own audit logging, which the pinned `convex` package declares as `audit(body)`; it is not taken as the record, because an entry in a log outside the database is not read by the operator's queries by operation, subject and request key, and whether such an entry proves a commit is not shown by the pinned package; a deployment may add it beside the tables as an export (E-43, D19) - [non-blocking] The retention and deletion policy's content is the owner's: which historical claims survive deletion and how long receipts, audit records and diagnostics are kept; the design gives the hooks and states no periods (D19) ## Rule - Logging and metrics failures never cancel a valid business write (D19, Sc ALL-1) - Mandatory business or security audit is written inside the transaction and fails closed (D19, Sc ALL-1) -- Telemetry that can veto a business write is not a standing requirement; a diagnostic sink is called after the business writes and its failure is swallowed and counted (D18, D19) +- Telemetry that can veto a business write is not a standing requirement; a diagnostic sink is called after the business writes, or in place of them when the command throws, and its failure is swallowed and counted (D18, D19) - Every bulk operation, sweeper and batch has a stated bound, tested against the pinned Convex version; platform limits are ceilings, not batch sizes (D19, F13) - Events stay small, and personal data sits behind references where rebuild does not need it (D19) - A retention and deletion policy exists before production data, and it states which historical claims survive deletion (D19) @@ -62,19 +62,20 @@ Security, bounded resource use, useful errors and tests start in Layer 1, so non - Security, bounded resource use, useful errors and tests start in Layer 1 (Thesis) - Every audit record of a command and every diagnostic record names its tenant, and every operator query over them takes the tenant scope; an `operatorAudit` record of a gate change names a tenant only through a tenant scope, and its query takes the scope (Law 11, D11, E-43) - Operator entries that read audit records are internal functions, so only admin access reaches them, as the operator entry contract of `spec:command.actor-and-scope` rules; each one over tenant data takes the tenant, so a client cannot read another tenant's audit (D11, Law 5, E-48) -- [extension] A diagnostic record is emitted per command through a sink that never throws, is bounded to 4 KiB, and holds identifiers, outcome kind, error code, versions and timings only (E-43, D19) +- [extension] A diagnostic record is emitted once for every execution of a command entry's handler, whether it returns an applied command, a business failure or a duplicate's stored outcome, or throws a rejection, a transient refusal or a technical failure, through a sink, by `emitDiagnostic`, which never throws; it is bounded to 4 KiB and holds identifiers, the outcome or the refusal with its code, the stream versions and two counts of work only, and no duration (E-43, D19) - [extension] An audit record is one document written by the command pipeline's step 10 through `writeAudit` for every declaration whose `audit` field is set, after an applied command and after a business failure alike, with every field taken from the command as `auditRecordSources` lists, and the failure of that write throws and rolls back the command (E-43, D19, E-38) +- [extension] No other ending of a command writes an audit record: a duplicate answered at step 5 writes none, the record of its original operation standing, and a rejection, a transient refusal or a technical failure leaves none, because the mutation that would hold it rolls back; whether a refused command needs a record anywhere is the question OQ2 asks, whose answer decides whether the conditional dispatcher of `spec:command.outcome-boundary` exists (E-43, OQ2, D7, D19) - [extension] An operator entry that changes the maintenance gate writes one `operatorAudit` record through `writeOperatorAudit` in the same mutation, with the stated operator, and the failure of that write throws and rolls the change back; the generation row and the restore's run record carry their operator themselves and write no audit record (E-43, E-48, E-8, D19) - [extension] Personal data that rebuild does not need lives in the `personalData` table and is referenced from events by a stable reference; deleting the row leaves the event's claim intact and its personal fields unreadable (E-43, D19) ## Design -Diagnostics and audit are two paths with opposite failure rules. `emitDiagnostic` runs at the end of the command's mutation, after the business writes, inside a try and catch that swallows every error and increments a counter in memory; nothing it does can throw into the mutation. `writeAudit` runs inside the mutation with the business writes, uses `ctx.db.insert` with a validator, and any failure, including a validator refusal or a document too large, throws and rolls the whole command back. Both are parent code; components neither log nor audit for the parent. +Diagnostics and audit are two paths with opposite failure rules. `emitDiagnostic` runs once for every execution of a command entry's handler: at the end of the mutation after the business writes when the command returns, and inside the entry's outermost catch when it throws. It catches every error its sink throws, writes one marked line with `console.error` in place of the lost record, and catches that too, so nothing it does can throw into the mutation. `writeAudit` runs inside the mutation with the business writes, uses `ctx.db.insert` with a validator, and any failure, including a validator refusal or a document too large, throws and rolls the whole command back. Both are parent code; components neither log nor audit for the parent. Optimistic-concurrency retries cannot be observed from inside a mutation, because a retried mutation re-runs from the start; the metric comes from the platform's function logs and insights. Rebuild progress is read from the generation's progress row and restore progress from the restore run row. The Layer 3 metrics are listed so the list is complete; they are measured once the obligation module is installed. -- transactionBoundary: audit is written inside the command's top-level mutation and fails closed; diagnostics are emitted from the same mutation after the business writes and never throw (D19, Sc ALL-1) -- convexSurface: `getAuditByOperation`, `getAuditBySubject`, `getAuditByRequestKey` as internal queries for operators, each taking `tenantId` first, and `getGateAudit` of `spec:application.write-pause` over `operatorAudit`; the diagnostic sink is `console.log` of one JSON line, read through the platform's logs (E-43, D19) +- transactionBoundary: audit is written inside the command's top-level mutation and fails closed; diagnostics are emitted from the same execution, after the business writes when it returns and in the entry's outermost catch when it throws, and never throw (D19, Sc ALL-1) +- convexSurface: `getAuditByOperation`, `getAuditBySubject`, `getAuditByRequestKey` as internal queries for operators, each taking `tenantId` first, and `getGateAudit` of `spec:application.write-pause` over `operatorAudit`; the default diagnostic sink, `consoleSink`, writes each record as one line with `console.log`, read through the platform's logs (E-43, D19) - tableAuditRecords: `auditRecords: defineTable({ tenantId: v.string(), operationId: v.string(), requestKey: v.optional(v.string()), commandType: v.string(), actor: actorValidator, subject: v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() }), kind: v.union(v.literal("security"), v.literal("business")), decision: v.union(v.literal("applied"), v.literal("businessFailure")), causedBy: causedByValidator, recordedAt: v.number() })` in `auditTables`, the schema fragment the library exports, where `causedByValidator` is the envelope's (E-43, D19, Law 11) - typeAuditRecordInput: `type AuditRecordInput = { tenantId: string; operationId: string; requestKey?: string; commandType: string; actor: Actor; subject: SubjectRef; kind: "security" | "business"; decision: "applied" | "businessFailure"; causedBy: CausedBy }`, the table's document without `recordedAt`, which `writeAudit` sets to the mutation's `Date.now()` (E-43, D19) - auditRecordSources: step 10 builds the input from the command alone: `tenantId`, `requestKey` and `actor` from the `PipelineCall`, `operationId` and `causedBy` from the `OperationRef` step 7 minted, `commandType` from the declaration's `name`, `kind` from the declaration's `audit.kind`, `decision` from the outcome's kind, and `subject` from the declaration's `permission.subjectFrom(input)` when it is declared and otherwise from the first entry of the outcome's `versions`, its `contextId`, `streamType` and `streamId`; a declaration that sets `audit`, declares no `subjectFrom` and returns no version throws a plain error, so the command fails closed (E-43, E-38, D19) @@ -84,28 +85,38 @@ Optimistic-concurrency retries cannot be observed from inside a mutation, becaus - indexAuditBySubjectUse: diagnosis by business subject, newest first (D19) - indexAuditByRequestKey: `.index("by_request_key", ["tenantId", "requestKey"])` (E-43) - indexAuditByRequestKeyUse: diagnosis by request key when the receipt has expired (D19, D6) -- typeDiagnosticRecord: `[extension] interface DiagnosticRecord { tenantId: string; operationId: string; requestKey?: string; commandType: string; actorKind: string; actorId: string; outcome: "applied" | "businessFailure" | "rejection" | "technicalFailure" | "transientRefusal"; errorCode?: string; versions: StreamVersion[]; causedBy?: string; durationMs: number; layer: number }` (E-43, D19) -- fnEmitDiagnostic: `emitDiagnostic(record: DiagnosticRecord, sink: DiagnosticSink = consoleSink): void` truncating to 4 KiB and swallowing every error the sink throws (E-43, D19, Sc ALL-1) +- typeDiagnosticRecord: [extension] `type DiagnosticRecord = { tenantId: string; commandType: string; namespace: CallerNamespace; kind: "applied" | "businessFailure" | "rejection" | "transient" | "technical"; code?: string; replayed: boolean; operationId?: string; requestKey?: string; correlationId?: string; actorKind?: ActorKind; actorId?: string; causedBy?: CausedBy; eventsAppended: number; readModelRows: number; versions: StreamVersion[] }`, where `kind` is the outcome, written `technical` for a technical failure, or `transient` for a transient refusal, which is not an outcome, as `classifyThrown` of `spec:command.outcome-boundary` names those two, and `causedBy` is the envelope's `CausedBy` (E-43, D19, D4) +- typeDiagnosticSink: [extension] `type DiagnosticSink = (line: string) => void` and `const consoleSink: DiagnosticSink = (line) => console.log(line)`; the pipeline passes the declaration's `diagnosticSink`, or `consoleSink` when the declaration names none (E-43, D19) +- diagnosticOnce: [extension] every execution of a command entry's handler emits exactly one record: at step 11, at step 5 before a duplicate is returned, or in the entry's outermost catch; a call that the entry's argument validators refuse runs no handler and emits none, its completion record in the function log being its trace; a record is an observation of one execution, written before the mutation ends, so an execution refused at commit by an optimistic-concurrency conflict keeps its line in its own completion record, the OCC rerun writes its own, and the receipt stays the authority for what committed (E-43, D19, F1) +- diagnosticSourcesReturned: [extension] for a command that returns at step 11, `tenantId`, `namespace`, `requestKey` and `correlationId` come from the `PipelineCall`, `actorKind` and `actorId` from its `actor.kind` and `actor.id`, `commandType` from the declaration's `name`, `kind` from the executor's outcome, `replayed` is false, `operationId` and `causedBy` come from the `OperationRef` step 7 minted, `eventsAppended` is the sum of `appended` over the executor's `streams` entries, `readModelRows` is the count `writeReadModels` returns at step 9, 0 when the declaration binds no read model, `versions` are the outcome's versions, and `code` is absent (E-43, D19, D2) +- diagnosticSourcesDuplicate: [extension] for a duplicate returned at step 5, the call's fields come as for a command that returns at step 11, with `causedBy` present only when the call states one, `kind` is the receipt's `outcome`, `replayed` is true, `operationId` and `versions` come from the receipt, `eventsAppended` and `readModelRows` are 0 because the execution did no work, and `code` is absent (E-43, D6, D19) +- diagnosticSourcesThrown: [extension] for a command that throws, the entry's outermost catch takes `tenantId`, `requestKey`, `correlationId` and, when the caller states one, `causedBy` from the entry's arguments, `namespace` from the internal entry's argument or `public` for the public entry, and `actorKind` and `actorId` only when an actor exists, the internal entry's argument or the actor the public entry's step 2 established; `commandType` comes from the declaration's `name`; `kind` and `code` come from `classifyThrown` of what the catch throws, `rejection` or `transient` with the error data's `code`, or `technical` with no `code`; `replayed` is false; `operationId` is absent, because nothing stored carries it; `eventsAppended` and `readModelRows` are 0 and `versions` is empty, because nothing committed (E-43, D4, D7, D19) +- fnDiagnosticLine: [extension] `diagnosticLine(record: DiagnosticRecord): string` returns `diagnostic ` followed by `JSON.stringify` of an object holding the record's fields in the order of `typeDiagnosticRecord`, whatever the order of the record's own keys, with absent optional fields left out and no other key; when that line measures more than `limitDiagnosticBytes` with `utf8Length`, `versions` is cut to its longest prefix for which the line fits, with the key `versionsOmitted`, the number of entries cut, placed just before `versions` and counted in the fit; when the line does not fit even with no entry kept, it is cut to its longest prefix of whole code points that measures at most `limitDiagnosticBytes` (E-43, D19) +- fnDiagnosticGapLine: [extension] `diagnosticGapLine(record: DiagnosticRecord): string` returns `diagnostic gap ` followed by `JSON.stringify` of `{ tenantId, commandType, kind }` from the record, with `operationId` after them when the record has one, cut as `diagnosticLine` cuts a line that does not fit (E-43, D19) +- fnEmitDiagnostic: [extension] `emitDiagnostic(record: DiagnosticRecord, sink: DiagnosticSink = consoleSink): void` calls `sink(diagnosticLine(record))` once inside a try; when that throws, whatever is thrown, it calls `console.error(diagnosticGapLine(record))` once inside a second try whose catch does nothing; it never throws, reads and writes no table, and itself writes no line but the gap line (E-43, D19, Sc ALL-1) +- diagnosticGapCount: [extension] each error `emitDiagnostic` swallows is counted by its one gap line, in the function log's completion record of the execution that lost its record, so the number of lost records is the number of lines that contain `diagnostic gap`; a native test reads them from the `logLines` of the completion records after a mark, and a `convex-test` or pure test from the calls of a spy on `console.error`; no counter is kept in memory, because module state is not documented to survive from one function call to the next (E-43, D19, Sc ALL-1) +- diagnosticNoDuration: [extension] the record carries no duration: `Date.now()` is fixed for the whole execution, as `recordedAtSource` of `spec:context.event-envelope` states, so two readings in one execution differ by nothing, and the backend records each execution's time on the execution's completion record in the function log, which also holds the execution's diagnostic line, so the design adds no second source of the same fact (E-43, D19) +- diagnosticModule: [extension] `DiagnosticRecord`, `DiagnosticSink`, `consoleSink`, `limitDiagnosticBytes`, `diagnosticLine`, `diagnosticGapLine` and `emitDiagnostic` live in `src/operations/diagnostic.ts` and are exported from `src/operations/index.ts`; the audit writers stay in `src/audit/` (E-43) - fnWriteAudit: `writeAudit(ctx: { db: GenericDatabaseWriter }, record: AuditRecordInput): Promise>` inserting the record with `recordedAt` through `ctx.db.insert` and letting every failure propagate; `AuditDataModel` is the data model of `auditTables` (E-43, D19, Sc ALL-1) - tableOperatorAudit: `operatorAudit: defineTable({ kind: v.union(v.literal("gate.close"), v.literal("gate.resume")), scopeKey: v.string(), reason: v.string(), generationId: v.optional(v.id("generations")), operator: v.string(), recordedAt: v.number() }).index("by_scope", ["scopeKey", "recordedAt"])` in `auditTables`; operational data of the deployment like the generation registry, so its index leads with the scope and not with a tenant, the deviation from Law 11's index rule that the registry records under E-8 (E-43, E-48, E-8, D19) - indexOperatorAuditByScopeUse: `getGateAudit`, who closed and who reopened one scope, newest first; no command reads the table (E-43, D19) - fnWriteOperatorAudit: `writeOperatorAudit(ctx: { db: GenericDatabaseWriter }, record: OperatorAuditInput): Promise>` inserting the record with `recordedAt` at the mutation's `Date.now()` and letting every failure propagate, where `type OperatorAuditInput = { kind: "gate.close" | "gate.resume"; scopeKey: string; reason: string; generationId?: Id<"generations">; operator: string }`; called by the write pause's `closeScope` and `resumeScope` and by nothing else (E-43, E-48, D19) -- faultInjection: `DiagnosticSink` and the audit table name are injectable in the fixture app only, so the acceptance test can break each subsystem; the production composition has no injection point; the gate's audit is broken without an injection point, by a `convex-test` schema whose `operatorAudit` table has a validator that refuses the record (Sc ALL-1, Acceptance scenarios, E-43) +- faultInjection: the two faults of Sc ALL-1 enter through the fixture composition's `PlaceOrder` alone and are keyed by data, so the shared libraries carry no test hook, no switch is read and the production composition has no injection point; that declaration sets `audit: { kind: "business" }` and `diagnosticSink: fixtureDiagnosticSink`; for the order ID `auditFaultOrderId`, `"order-audit-fault"`, its `subjectFrom` returns the order's subject with one more field, `auditFault: true`, which the `auditRecords` validator does not declare, so Convex refuses step 10's insert, while authorization compares only the subject's three fields and passes; `fixtureDiagnosticSink` throws `new Error("Fault injected: the diagnostic sink is broken")` for every line that contains `"tenantId":"tenant-sink-fault"`, the tenant `sinkFaultTenant`, and passes every other line to `consoleSink`; `auditFaultOrderId`, `sinkFaultTenant` and `fixtureDiagnosticSink` are exported from `fixture/convex/orders.ts`; the gate's audit is broken without an injection point, by a `convex-test` schema whose `operatorAudit` table has a validator that refuses the record (Sc ALL-1, Acceptance scenarios, E-43, E-15) - tablePersonalData: `personalData: defineTable({ tenantId: v.string(), ref: v.string(), subject: v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() }), fields: v.any(), createdAt: v.number(), deletedAt: v.optional(v.number()) }).index("by_ref", ["tenantId", "ref"]).index("by_subject", ["tenantId", "subject.contextId", "subject.streamType", "subject.streamId"])` (E-43, D19, Law 11) - personalDataReference: an event payload that would hold personal data rebuild does not need holds `personalRef` instead; a deletion sets `deletedAt` and clears `fields`, and the retention policy states which claims the event still makes afterward (E-43, D19) - retentionHooks: receipts expire by the receipt contract, audit records and diagnostics by the retention policy, personal data by deletion requests; the design sets no period (D19, D6) - diagnosisPath: request key to receipt, operation ID to events by the journal's operation index and to audit by operation, subject to audit by subject, causation through `causedBy` on events and audit records (D19, D2) -- metricCommandLatency: duration of each top-level mutation, from the diagnostic record's `durationMs` and the platform's function logs (D19) +- metricCommandLatency: duration of each execution of a top-level mutation, from the time the backend records on its completion record in the function log, `executionTime` on the local backend (D19) - metricOccRetries: optimistic-concurrency retries per function, from the platform's logs and insights, since a mutation cannot observe its own re-execution (D19, F1) -- metricEventWrites: events appended per command, from the versions in the diagnostic record (D19) -- metricReadModelCost: read-model rows written per command and documents read per query, from the first experiment's measurements and the diagnostic record (D19, D8) +- metricEventWrites: events appended per command, from the diagnostic record's `eventsAppended` (D19) +- metricReadModelCost: read-model rows written per command, from the diagnostic record's `readModelRows`, and documents read per query, from the first experiment's measurements (D19, D8) - metricOldestUnresolvedObligation: age of the oldest unresolved obligation, measured once Layer 3 is installed (D19, D13) - metricRetryAndExhaustion: retry and exhaustion counts, measured once Layer 3 is installed (D19, D13) - metricProviderUncertainty: obligations in needs attention for an ambiguous provider outcome, measured once Layer 3 is installed (D19, D14) - metricRebuildProgress: batches done, rows written and misses per generation, from the generation's `generationProgress` row in the registry (D19, D9, E-40) - metricRestoreProgress: checks passed, examined counts and findings per restore run, from the restore run record (D19, Sc L2-8) - releaseRule: a release keeps a building generation's cursor, a retired generation's rows and an import's progress row readable; a schema change to any of them ships with a migration that runs before the new code reads them (D19) -- limitDiagnosticRecord: 4 KiB per diagnostic record; larger records are truncated, never dropped (E-43, D19) +- limitDiagnosticRecord: `limitDiagnosticBytes`, 4,096 bytes of UTF-8 per line that `diagnosticLine` or `diagnosticGapLine` returns, measured with `utf8Length`; a longer record is cut as `fnDiagnosticLine` states, never dropped (E-43, D19) - limitOperatorQuery: audit queries are paginated with at most 100 records per page (E-43, F13) ## Example space @@ -122,5 +133,8 @@ And the failure is {surfaced:"reported as a diagnostic gap without touching the - A reviewer confirms that `emitDiagnostic` is the only path from a command to logs or metrics, that it cannot throw into the mutation, and that no code path decides a business outcome from a diagnostic. - A reviewer confirms that the command pipeline's step 10 calls `writeAudit` for every declaration whose `audit` field is set, inside the mutation, and that no catch surrounds it. +- A pure test confirms `diagnosticLine`'s prefix, key order and both cuts, `diagnosticGapLine`'s fields, and that `emitDiagnostic` with a sink that throws writes exactly one gap line with `console.error` and nothing else and returns, also when `console.error` throws. +- A `convex-test` test confirms exactly one record per execution, each field from its source, read through a sink the test gives the declaration or through a spy on `console.log` where the sink is `consoleSink`: for an applied command and a business failure at step 11, a duplicate at step 5, and a rejection, a transient refusal and a technical failure through `relayFailure`, and no record for a call its argument validators refuse. +- A `convex-test` test confirms that `eventsAppended` and `readModelRows` count the work done: one event and two events appended to a stream at version 100, and step 9 writing zero, one and two generation rows. - A reviewer confirms that every batch in the corpus states its bound and that every index of `auditRecords` and of personal data leads with the tenant, and that the `operatorAudit` index leads with the scope. - A reviewer confirms that a retention and deletion policy document exists before any production data and that it names which claims survive deletion. From 2140eb7c74c612f1224ee401f9d0fe39c7f46f22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:52:27 +0200 Subject: [PATCH 123/155] design: 4c the history view on paper The history fold gives one row whatever the order across streams, keeps its row past the subject's end and drops baselines; the paused path names its source in the cursor, verifies by repeating the backfill and takes its batch size from the one formula; rollback of a paused view is a refusal on every path; the cross-context row is narrowed to D8's parent query; the aggregate write splits on a key change and derives its entity key from the source stream. The order allocation history's pure rules and a pure test of their order independence. Co-Authored-By: Claude Opus 5.5 --- .../specs/application/first-experiment.sdp.md | 2 +- .../application/generation-registry.sdp.md | 9 +- .../orders-inventory-example.sdp.md | 2 +- .../application/projection-contract.sdp.md | 56 +++-- design/specs/application/read-models.sdp.md | 17 +- design/specs/application/rebuild.sdp.md | 37 ++-- ...build.write-pause-rebuild-interrupt.sdp.md | 8 +- design/specs/application/write-pause.sdp.md | 1 + example/domain/orderAllocationHistory.ts | 116 ++++++++++ tests/pure/order-allocation-history.test.ts | 205 ++++++++++++++++++ 10 files changed, 400 insertions(+), 53 deletions(-) create mode 100644 example/domain/orderAllocationHistory.ts create mode 100644 tests/pure/order-allocation-history.test.ts diff --git a/design/specs/application/first-experiment.sdp.md b/design/specs/application/first-experiment.sdp.md index 8a88c4a..e230cb9 100644 --- a/design/specs/application/first-experiment.sdp.md +++ b/design/specs/application/first-experiment.sdp.md @@ -116,7 +116,7 @@ The experiment is an application, a test suite and a report. The application is - contentionShape: N `PlaceOrder` commands for the same stock item sent concurrently, at N of 2, 8 and 32, with stock for one, so that one is applied and the rest are rejected for short stock or failed by the engine after its bounded retries, which is Sc L1-11's shape widened into the contention run; the run reports bytes read per call beside documents, the engine's retries where the backend exposes them, and how many commands the engine failed after its bounded retries, which the outcome boundary classifies as technical failures the caller may retry; the shared stock item is the first line of every order and each other line is on a stock item of that command alone, received once with quantity two, and no caller retries during the cell, while the caller may repeat a command the engine failed under its same request key (First experiment, Sc L1-11, F1, E-5, E-47) - limitOrderLines: 100 lines per `PlaceOrder`, the largest order the example supports and the bound its declaration carries, chosen under the 1 second timeout, the F13 ceilings and the adapter's stream and byte bounds; it is the example's promise and no platform limit, and the experiment reports documents read and written per line (OQ3, F13) - limitStockItemIdBytes: [extension] 64 bytes of UTF-8 for the stock item ID of each order line of `PlaceOrder` and each item of `ReceiveStock`, the bound both declarations carry and check in their refinements, so a longer one is refused `invalidInput` at step 1 of the command pipeline, before any read and before any context is called, as the stock item ID bound of `spec:application.orders-inventory-example` states; it is chosen so that the maximum order's `OrderPlaced` fits the 16,384-byte payload bound of E-12: `getConvexSize` measures the payload at 25 + 100 × (53 + 64) = 11,725 bytes at the bound, where 128 bytes would make 18,125 and the 256 bytes of a stream ID 30,925 (E-46, E-12, OQ3, F13) -- limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) +- limitExpectedDocuments: the design expects one receipted `PlaceOrder` of N lines, each on its own stock item, to write 2N + 4 documents, one stock item state and one event per line, the order state, the order event, the receipt and the summary row, and apart from them the audit record's one document; it expects the command to read three documents per line, the stock item's state to decide against, the journal's event at the expected stream version to check the append against, and the state again when it is replaced by its document ID, which the backend charges as a read, and beside them the caller's grants, the registry row and the gate; these counts are of `PlaceOrder` with the order summary its one read model, and binding the order allocation history adds one document written, its row, and one read, its registry row, since the lookup of a new order's row finds nothing; `observedPlaceOrderDocuments` holds what the pinned backend counted (E-47, F13, D10) - observedPlaceOrderDocuments: [extension] on the local backend `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, four grants, one active generation, no audit record written and no gate document, the `usageStats` of one receipted `PlaceOrder` of N lines, each line on its own stock item received once, counted 3N + 5 documents read and 2N + 4 written at N = 1, 10 and 100: 8 read and 6 written, 35 and 24, 305 and 204; at N = 10 the reads are the ten stock item states loaded, the ten journal events at the expected stream versions, the ten replacements of those states by document ID, the four grants and the registry row; the lookups of the receipt, the order stream, its events and the summary row that find nothing read no document, a second earlier event on a stock item adds no read, and two of the ten lines on one stock item read three documents fewer and write two fewer; the gate read of the pipeline's step 7 runs in every one of these commands, and with no gate document in the table it finds nothing and adds no document read and no byte, so the counts above include it, and no measurement here has a gate document in the table (E-47, F13, E-8) - observedMeasurementPins: [extension] the measurement records name the native backend tier, the local backend `precompiled-2026-09-28-5c7cb5b`, `convex` 1.46.0, `convex-helpers` 0.1.124, `convex-test` 0.0.60, Node v24.19.0 and Linux 7.2.7-200.fc44.x86_64 on x64; these observations concern that local environment only (E-47, E-15, Sc L2-9) - observedMeasurementBytes: [extension] at the native backend tier with `observedMeasurementPins`, fixed seven-character stock and order IDs, four grants and one active generation, orders of 1, 10 and 100 distinct stock items received once read 2848, 14359 and 129469 bytes and write 4280, 18419 and 159809 bytes; the corresponding write-index rows are 25, 97 and 817; a ten-line authorized duplicate or changed-input conflict reads 5 documents and 3441 bytes and writes zero, while short stock or a missing journal tail on the last line reads 32 documents and 13106 bytes and writes zero; the read of the active generation's `generations` row, which no longer carries the batch counts, is 95 bytes smaller than before, and each order summary row writes 9 bytes more for `key` in its `by_status` entry (E-47, F13) diff --git a/design/specs/application/generation-registry.sdp.md b/design/specs/application/generation-registry.sdp.md index 1bad472..bc89f3f 100644 --- a/design/specs/application/generation-registry.sdp.md +++ b/design/specs/application/generation-registry.sdp.md @@ -43,11 +43,11 @@ The registry is two parent tables. The `generations` table records every generat - Every generation of a read model has one row keyed by read model name and generation number, and generation numbers increase by one per read model, starting at 1 (E-8, D9) - A generation is in exactly one of `building`, `verifying`, `verified`, `active`, `retired`, `aborted` or `purged` (E-8, D9) - At most one generation per read model is `active`, and at most one is in `building`, `verifying` or `verified`; these three states are the in-flight states (E-8, D9) -- The allowed transitions are none to building by `startGeneration`, building to verifying by the backfill batch that ends the enumeration, verifying to verified by the verify batch that ends its pass, verified to active by `switchGeneration`, active to retired by the same `switchGeneration`, retired to verifying by `rollbackGeneration`, building, verifying or verified to aborted by `abortGeneration`, and retired or aborted to purged by the purge batch that ends its pass; no other transition exists, and none leads from nothing to active; `interruptGeneration` and `resumeGeneration` change no state, and `resumeGeneration` continues the pass the progress row names, a started purge of a `retired` or `aborted` generation included, at the batch size the progress row holds, since it takes no batch size (E-8, D9, E-40) +- The allowed transitions are none to building by `startGeneration`, building to verifying by the backfill batch that ends the enumeration, verifying to verified by the verify batch that ends its pass, verified to active by `switchGeneration`, active to retired by the same `switchGeneration`, retired to verifying by `rollbackGeneration` for a generation without `pauseRequired`, building, verifying or verified to aborted by `abortGeneration`, and retired or aborted to purged by the purge batch that ends its pass; no other transition exists, and none leads from nothing to active; `interruptGeneration` and `resumeGeneration` change no state, and `resumeGeneration` continues the pass the progress row names, a started purge of a `retired` or `aborted` generation included, at the batch size the progress row holds, since it takes no batch size (E-8, D9, E-40) - An operator entry of `spec:application.rebuild` makes the transitions named after it, a scheduled batch makes the other three, and nothing else writes a generation row (E-8, E-48, D19) - [extension] `startGeneration` refuses, and so makes no transition, while the tenant list of `spec:command.actor-and-scope` holds no row and the `grants` table holds one, with a message that names `fillTenants`, and while the tenant fill of `spec:application.rebuild` is in its pass `fill`, because a generation that walked an empty or half-filled list on a deployment whose grants predate the list would be verified and switched without the rows of the tenants it missed (E-49, E-8, Law 11) - [extension] A read model's first generation is generation 1 in `building`, started by `startGeneration` like any other, and the read model has an active generation only after that generation is verified and switched; on a deployment with no subject the build is one backfill batch and one verify batch for each tenant of the tenant list, and one of each when the list is empty (E-8, D9) -- A command that maintains a read model writes the active generation and the in-flight generation if one exists; a retired generation is kept unchanged until it is purged and never written, and a rollback reopens it as `verifying` so that live commands write it again while its verify pass repairs it (D9, E-8) +- A command that maintains a read model writes the active generation and the in-flight generation if one exists; a retired generation is kept unchanged until it is purged and never written, and a rollback reopens it as `verifying` so that live commands write it again while its verify pass repairs it; a generation with `pauseRequired` is never reopened, because nothing would close its sources while live commands fold onto rows that missed the events since its retirement (D9, E-8) - [extension] Each generation is written with the projection whose version its row records, on the live path and by its batches alike, so the rows of one generation are one truth of one projection version (E-9, E-8, D8) - A query reads the active generation and nothing else; a query over a read model that has no active generation throws a plain error that names the read model, a technical failure, and never answers an empty page (D9, E-8, Law 6) - [extension] The generation row carries the state, the projection version, the fence and who started and last changed it, and nothing a batch writes on its healthy path except the state that the batch ending a pass sets; the progress row carries the cursor, the batch size and the counts, the batch size set by `startGeneration` and never changed after, so every pass of the generation, a resumed one and its purge included, runs at it, a purge batch capped further by `limitPurgeBatch` of `spec:application.rebuild`, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) @@ -65,18 +65,19 @@ Tables live in the parent's `schema.ts`, spread from the library's `readModelTab - validatorGenerationState: `const generationStateValidator = v.union(v.literal("building"), v.literal("verifying"), v.literal("verified"), v.literal("active"), v.literal("retired"), v.literal("aborted"), v.literal("purged"))` (E-8, D9) - tableGenerations: `generations: defineTable({ readModel: v.string(), generation: v.number(), projectionVersion: v.number(), state: generationStateValidator, pauseRequired: v.boolean(), fence: v.number(), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), switchedAt: v.optional(v.number()), retiredAt: v.optional(v.number()), retireAfter: v.optional(v.number()) })` where `changedAt` and `changedBy` are the time and the operator of the last operator entry that changed the row, which no batch writes, `switchedAt` is when the generation last became active, and `retiredAt` and `retireAfter` are set while it is `retired` and absent otherwise (E-8, E-48, D9, F13) +- generationPauseRequired: `pauseRequired` is true on every generation of a history view, set by `startGeneration` from its rebuild target and changed by nothing, and false on every other; `switchGeneration` and `abortGeneration` reopen the source scopes such a generation closed, and `rollbackGeneration` refuses it (E-8, D9) - indexGenerationsByReadModel: `.index("by_read_model", ["readModel", "generation"])` (E-8) - indexGenerationsByReadModelUse: the next generation number at `startGeneration`, read as the last row in descending order, and the operator listing `getGenerations` (E-8, D9) - indexGenerationsByReadModelState: `.index("by_read_model_state", ["readModel", "state"])` (E-8) - indexGenerationsByReadModelStateUse: `generationsToWrite` in every command that maintains the read model, `activeGeneration` in every query over it, and the in-flight check of `startGeneration`, `switchGeneration` and `rollbackGeneration` (D9, E-8) -- validatorBatchCursor: `const batchCursorValidator = v.object({ tenantId: v.string(), pageCursor: v.union(v.string(), v.null()), streamId: v.optional(v.string()), eventCursor: v.optional(v.union(v.string(), v.null())) })` the position of a pass, the tenant being iterated and the `paginator` cursor within it, `null` at the start of a tenant; `streamId` and `eventCursor` are set only by the batches of a history view (E-8, E-40, Law 11) +- validatorBatchCursor: `const batchCursorValidator = v.object({ tenantId: v.string(), pageCursor: v.union(v.string(), v.null()), sourceIndex: v.optional(v.number()), streamId: v.optional(v.string()), eventCursor: v.optional(v.union(v.string(), v.null())) })` the position of a pass, the tenant being iterated and the `paginator` cursor within it, `null` at the start of a tenant or of a source; `sourceIndex`, the position in a history view's sources, `streamId` and `eventCursor` are set only by the batches of a history view, whose pass goes tenant, then source, then stream, then event page (E-8, E-40, Law 11) - validatorProgressPass: `const progressPassValidator = v.union(v.literal("backfill"), v.literal("verify"), v.literal("purge"), v.literal("idle"))` naming the pass a batch chain is driving, `idle` when none is (E-40) - tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), pass: progressPassValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position in the pass that `pass` names and `null` before that pass has read its first tenant; `batchesDone` counts every batch of the generation that reached its checkpoint, the batch that ends a pass included and a parked batch not, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted; every write of the row sets `updatedAt` to the writing mutation's `Date.now()` (E-40, D9, F13) - indexProgressByGeneration: `.index("by_generation", ["generationId"])` (E-40) - indexProgressByGenerationUse: the one read a batch and an operator entry make to find a generation's progress row; no command and no query over a read model reads the table (E-40, F1) - tableProjectionMarkers: `projectionMarkers: defineTable({ tenantId: v.string(), readModel: v.string(), generation: v.number(), entityKey: v.string(), aggregateKey: v.string(), contribution: v.number(), sourceVersions: v.array(streamVersionValidator) })` (E-8, D9, Law 11) - indexMarkersByEntity: `.index("by_entity", ["tenantId", "readModel", "generation", "entityKey"])` (E-8) -- indexMarkersByEntityUse: point read by `applyAggregate` before every aggregate write (D9, E-8) +- indexMarkersByEntityUse: point read by `applyAggregate` before every aggregate write, with the `entityKey` it derives from the entity's source stream, `JSON.stringify([contextId, streamType, streamId])` of the input's stream version, as `spec:application.projection-contract` states (D9, E-8, E-9) - typeRegistryReader: `type RegistryReader = { db: GenericDatabaseReader }` and `type RegistryWriter = { db: GenericDatabaseWriter }`, the part of a ctx the registry's helpers use, where `ReadModelDataModel` is the data model of `readModelTables`, the schema fragment that holds `generations`, `generationProgress`, `projectionMarkers` and the tenant fill's `tenantFill` row of `spec:application.rebuild`; every helper that takes them is generic over the composition's data model, because a database reader is not covariant in its data model (E-8) - typeWritableGeneration: `type WritableGeneration = { role: "active" | "building" | "verifying"; generation: number; projectionVersion: number }` where the role is `active` for the state `active`, `building` for the state `building`, and `verifying` for the states `verifying` and `verified` (E-8, E-9, D9) - fnGenerationsToWrite: `generationsToWrite(ctx: RegistryReader, readModel: string): Promise` making at most two indexed reads, the ranges of `by_read_model_state` from `active` to `building` and from `verified` to `verifying`, throwing `"Read model " + readModel + " has " + a + " active generations and " + b + " in building, verifying or verified"` when they hold two active generations or two in-flight ones, and returning one entry per generation a command writes, the active generation first when there is one, then the in-flight one, and an empty list when the read model has neither; the answer is the same for every tenant (E-8, D9) diff --git a/design/specs/application/orders-inventory-example.sdp.md b/design/specs/application/orders-inventory-example.sdp.md index 471384b..fc111e4 100644 --- a/design/specs/application/orders-inventory-example.sdp.md +++ b/design/specs/application/orders-inventory-example.sdp.md @@ -66,7 +66,7 @@ The second lifecycle command is `CancelOrder`. The stock item keeps two totals a - **stock limit rejection** — The rejection `stockLimitExceeded` that the stock item decider returns, and the Inventory context's adapter throws, for a receive that would take the quantity on hand past the largest safe whole number, with the message "Cannot receive N on top of the M on hand" and the details `{ quantity, onHand }`; `ReceiveStock` lists the code beside `invalidQuantity`, and nothing commits (D4, D7, E-46). - **stock contention** — N `PlaceOrder` commands for the same stock item sent concurrently, as the first experiment's contention run sends them at N of 2, 8 and 32 with stock for one; on the native backend one is applied and every other is rejected `insufficientStock` or, past the engine's bounded retries, failed as a technical failure the caller may repeat, and the experiment records the retry cost of the losers apart from the healthy path (Sc L1-11, First experiment). - **order summary** — The essential summary: a per-entity read model of the parent with the order's status, line count, total and the time it was placed, `{ orderId, status: "placed" | "cancelled", lineCount, total, placedAt }`, written from the order's DTO by every command that writes the order, so `PlaceOrder` inserts the row with `placed` and `CancelOrder` replaces it with `cancelled`, and listed by status (E-46, D8). -- **order allocation history** — A history-dependent cross-stream read model of the parent keyed by order ID, folded as a `HistoryProjection` from `OrderPlaced` in Orders and `StockAllocated` and `AllocationReleased` across the stock item streams the order touched, holding per line when it was allocated and released; it is bound once per source stream type, rebuilds only under the write pause, and exists in the experiment for Sc L2-6, not as an essential read (E-9, E-46, D9, Sc L2-6). +- **order allocation history** — A history-dependent cross-stream read model of the parent keyed by order ID, folded as a `HistoryProjection` from `OrderPlaced` and `OrderCancelled` in Orders and `StockAllocated` and `AllocationReleased` across the stock item streams the order touched, holding when the order was placed and cancelled and, per stock item, the quantity allocated, when, and when it was released; each of its fields is written by the events of one stream, so its fold gives one row whatever the order across streams, a stock event before `OrderPlaced` included, and its row is kept once the order is cancelled, so a release after the cancel folds into it; it is bound once per source stream type, rebuilds only under the write pause, and exists in the experiment for Sc L2-6, not as an essential read (E-9, E-46, D9, Sc L2-6). - **journal inspection** — The context queries that return a stream's events and its current version for a reviewer, used by the experiment and the restore checks (First experiment, D2). - **maximum order size** — The largest number of lines `PlaceOrder` accepts, 100, the bound its declaration carries: the example's promise and no platform limit, chosen by the command's author under the adapter's stream and byte ceilings (OQ3, E-46). - **stock item ID bound** — The longest stock item ID the example accepts, 64 bytes of UTF-8, the bound of every command that names a stock item, which its declaration carries and checks in its refinement: `PlaceOrder` on each order line, beside its 100 lines, and `ReceiveStock` on each item it receives, because a stock item whose ID no order may name could never be ordered and has no use; a command whose line or item has a longer stock item ID is refused `invalidInput` at step 1 of the command pipeline, before any read and before any context is called, with the message "Line I needs a stock item ID of at most 64 bytes of UTF-8, not N" and the details `{ line, length, limit }` of the first such entry, `line` being its position from 0 in the order's lines or the received items; the bound exists because `OrderPlaced` carries every line and `append` holds its payload to the 16,384 bytes of the events-stay-small bound: `getConvexSize` measures the payload `{ lines, total }` at 25 bytes plus, for each line, 53 bytes and its stock item ID's UTF-8 bytes, every number counting 9 bytes whatever its value, so 100 lines at 64 bytes make 25 + 100 × (53 + 64) = 11,725 bytes and fit, while 100 lines at 128 bytes would make 18,125 bytes and at the 256 bytes a stream ID may have 30,925, which the Orders context's `append` refuses as a technical failure inside the Orders call; 64 is therefore the largest power of two at which the maximum order fits (OQ3, E-12, E-46). diff --git a/design/specs/application/projection-contract.sdp.md b/design/specs/application/projection-contract.sdp.md index e1c59ce..def3f02 100644 --- a/design/specs/application/projection-contract.sdp.md +++ b/design/specs/application/projection-contract.sdp.md @@ -19,7 +19,7 @@ relations: Story · Detail: full · Traces: D8, D9, Law 6, Law 10, Law 11, F1, F13, E-8, E-9. -A projection is the named, versioned, deterministic function that turns a context's DTO at a stream version into a read-model row, and `applyProjection` is the one helper that writes that row, shared by the use case's live update and the rebuild's backfill. The contract pins the projection's shape, the declaration of a per-entity read model, which names its table and row budget and holds its projection, the row conventions every read model follows, the write rule that decides between insert, update and skip, and the aggregate form with its per-entity marker. Everything here is an extension of D8 and D9: the doc fixes the properties, this contract fixes the signature. +A projection is the named, versioned, deterministic function that turns a context's DTO at a stream version into a read-model row, and `applyProjection` is the one helper that writes that row, shared by the use case's live update and the rebuild's backfill. The contract pins the projection's shape, the declaration of a per-entity read model, which names its table and row budget and holds its projection, the row conventions every read model follows, the write rule that decides between insert, update and skip, the aggregate form with its per-entity marker, and the history form that folds the events of several streams into one row. Everything here is an extension of D8 and D9: the doc fixes the properties, this contract fixes the signature. ## Intent @@ -30,9 +30,12 @@ A projection is the named, versioned, deterministic function that turns a contex ### Open questions -- [non-blocking] Extension E-9: the doc says a projection is named, versioned and deterministic and that live update and rebuild share its logic; it does not give the signature or the row shape, for counts and sums it names the marker but not the aggregate row, and for the read models that depend on history across several streams, the ones the write pause exists for, it names no input. The option taken here: a per-entity read model declared as a `ReadModel` with `name`, `table`, `rowBudgetBytes` and its `projections`, a list of `Projection` with the latest first, each holding `version`, `keyOf` and `project` alone, rows with `tenantId`, `generation`, `key`, `projectionVersion` and `sourceVersions`, one `applyProjection` helper with a mode per caller that writes each generation by its role, active, building or verifying, with the projection whose version the generation records, for an `AggregateProjection` an aggregate row with `value` and `entityCount` and no `sourceVersions`, because the markers carry them, written by `applyAggregate` under the marker's version compare with the mode ignored, and for a history-dependent view a `HistoryProjection` that folds events into the prior row, fed on the live path by the events the operation appended, which the adapter's `streams` entry now carries, and on the backfill by the context's `history` pages under the write pause, with `sourceVersions` recording the highest version folded per stream so that an event is never folded twice; a cross-context or cross-stream row is built by one projection bound once per source stream type; the owner confirms the three forms and has not ruled on the split of a per-entity read model into `ReadModel` and `Projection`, on the write by generation role or on the row measured against its budget at every write (E-9, E-23, E-24, D8, D9) +- [non-blocking] Extension E-9: the doc says a projection is named, versioned and deterministic and that live update and rebuild share its logic; it does not give the signature or the row shape, for counts and sums it names the marker but not the aggregate row, and for the read models that depend on history across several streams, the ones the write pause exists for, it names no input. The option taken here: a per-entity read model declared as a `ReadModel` with `name`, `table`, `rowBudgetBytes` and its `projections`, a list of `Projection` with the latest first, each holding `version`, `keyOf` and `project` alone, rows with `tenantId`, `generation`, `key`, `projectionVersion` and `sourceVersions`, one `applyProjection` helper with a mode per caller that writes each generation by its role, active, building or verifying, with the projection whose version the generation records, for an `AggregateProjection` an aggregate row with `value` and `entityCount` and no `sourceVersions`, because the markers carry them, written by `applyAggregate` under the marker's version compare with the mode ignored, and for a history-dependent view a `HistoryProjection` that folds events into the prior row, fed on the live path by the events the operation appended, which the adapter's `streams` entry carries, and on the backfill by the context's `history` pages under the write pause, with `sourceVersions` recording the highest version folded per stream so that an event is never folded twice; a row that derives from more than one source stream type is a history row, its projection bound once per source stream type, and a small cross-context view of current state is D8's parent query over bounded component reads; the owner confirms the three forms and has not ruled on the split of a per-entity read model into `ReadModel` and `Projection`, on the write by generation role or on the row measured against its budget at every write (E-9, E-23, E-24, D8, D9) - [non-blocking] Extension E-9, two projection versions: the doc says live update and rebuild share one projection and does not say how a command writes two generations built by two versions; the option taken here is that a read model declares its projection versions as a list, the latest first, and `applyProjection` writes each generation with the projection whose version the generation row records, failing as a technical failure when the read model declares none of that version, so the rows of one generation are one truth of one version; the cost is that an earlier version stays in the code while a generation that records it is written or can be rolled back to, and a projection changed without a version change is still undetected (E-9, D8, D9) - [non-blocking] The rule that a row with a newer version is never overwritten is read as the backfill path's rule only, so a live write to the active generation always lands, as `step3` has it; the owner has not confirmed that reading (D9, E-9) +- [non-blocking] Extension E-9, the history fold: D2 orders events only within a stream, and the two paths feed one history row in different orders, the live path command by command and the backfill stream by stream; the option taken here is a fold that gives one row whatever the order across streams and however its events are split between calls, a row no fold deletes, which keeps its subject's ended state, and baselines dropped before `keyOf`, so a view whose meaning needs the order across streams waits for the ordered consumer of D18 (E-9, D2, D9, D18, E-25) +- [non-blocking] Extension E-9, a composite row of current state: D8 lets a small cross-context view be a read model updated in the command, and no form here writes one row from the DTOs of two sources, because `project` builds a whole row from one DTO and a backfill enumerates one source; the option taken here is D8's other default, a parent query over bounded component reads, until a view needs such a row, which then needs a form whose `project` takes the DTO of every matching entry of one command and whose backfill reads the other context in the same mutation, online (E-9, D8, E-45) +- [non-blocking] Extension E-9, the history form's one version: a `HistoryProjection` declares one version, so a deploy that changes it leaves the active generation, which records the earlier version, unwritable, and every command whose events feed the view fails as a technical failure until a generation of the new version is switched; the option taken here is that one version, because a view rebuilt under the pause is never written by two versions at once, and a list of versions like the `projections` of a `ReadModel` is the form a view takes when its sources must stay writable across such a deploy (E-9, D8, D9, Law 6) ## Contract @@ -41,8 +44,8 @@ A projection is the named, versioned, deterministic function that turns a contex - Live update and rebuild call the same `applyProjection` with the same read model, and each generation is written by the projection of the version it records; only the mode differs (D8, D9) - The pipeline's step 9 picks the live-created mode when the `created` flag on the operation outcome's `streams` entry for that stream is true, which the adapter sets from a load that found no row, and the live-updated mode otherwise; no use case derives the flag (D9, E-9, E-23) - The DTO a projection receives is the same `toDto(state, meta)` on both paths, the adapter's `streams` entry on the live path and the context's `list` page on the backfill, and it carries the stream's `StreamVersion` as `version`, so no adapter between two DTO shapes exists (D8, D9, E-9, E-24) -- Every read-model row carries `tenantId`, `generation`, `key` and `projectionVersion`, a per-entity or cross-context row carries `sourceVersions` as well, and every read-model table has the `by_key` index leading with the tenant (E-9, Law 11, D9) -- `sourceVersions` holds one `StreamVersion` per source stream the row derives from; a per-entity row holds one, a cross-context row holds one per context (E-9, D2, D8) +- Every read-model row carries `tenantId`, `generation`, `key` and `projectionVersion`, a per-entity or history row carries `sourceVersions` as well, and every read-model table has the `by_key` index leading with the tenant (E-9, Law 11, D9) +- `sourceVersions` holds one `StreamVersion` per source stream the row derives from; a per-entity row holds one, and a history row one per stream whose events it folded (E-9, D2, D8) - [extension] An aggregate row derives from every entity under its key, so it carries no `sourceVersions`; it carries `value`, the sum of the recorded contributions, and `entityCount`, the number of markers, and the per-entity marker carries the `sourceVersions` the row conventions would otherwise hold (E-9, D9) - [extension] A read model declares its name, the parent table that holds its rows, `rowBudgetBytes`, the largest row it writes, 16 KiB by default and at most `limitRowSize`, and its projections, one per version it can still write, the latest first, with distinct versions and at least one; every version writes rows the one table's validator accepts, and a change of shape the table cannot hold beside the earlier one is a new read model; a projection is the versioned rules alone, so they are tested without a table; the rebuild derives its batch bound from the declared budget the way the context derives its page from the stream budget (E-9, E-40, F13) - On the backfill path a row is written only if it is missing or its `sourceVersions` are older than the input's for the same stream, and a row with a newer version is never overwritten; the rule is the backfill path's only, and a live write to the active generation always lands (D9) @@ -52,10 +55,14 @@ A projection is the named, versioned, deterministic function that turns a contex - In the backfill mode the one generation the batch names is written whatever its role: the row is inserted if missing, updated if older, and, when the DTO projects to `null`, deleted if its `sourceVersions` are older than the input's and left as `skipped-newer` otherwise, which is how a verify pass makes absence right as well as presence (D9, E-8, E-9) - A projection may return `null` for a DTO that has no row, such as a deleted subject; `applyProjection` then deletes the row if present, in the backfill mode only when the row is older than the input, and does nothing when it is absent, and nothing records the deletion, because a backfill batch reads the source in the same mutation that writes the row, so a batch that read the subject before its deletion committed re-runs under optimistic concurrency and reads the deleted subject, as the rebuild's `occRace` states (E-9, D2, F1) - An aggregate projection contributes a number per entity to an aggregate row and keeps a per-entity marker so live commands and backfill never count one entity twice (D9) -- [extension] `applyAggregate` ignores the mode: the marker's `sourceVersions` decide, a marker that is not older than the input makes the write `skipped-newer`, an older or missing marker is written with the new contribution and versions and the difference goes to `value`, and a `null` contribution removes the marker and its recorded contribution; a live update on an entity backfill has not reached is therefore not partial, because `contribute` sees the whole DTO, which is why the missing-row rule of the per-entity path does not apply (E-9, D9, F1) -- [extension] A history projection folds events into a row: `keyOf` maps each event to the row it belongs to or to none, and `project` folds the events of one key, in stream-version order within each stream, into the prior row; it is deterministic in the prior row and the events, reads nothing else, and is the one form a read model that depends on history across several streams may take, which is why such a read model rebuilds only under the write pause (E-9, D8, D9, Law 10) -- [extension] A history row's `sourceVersions` hold, per source stream, the highest event version folded into it; `applyHistoryProjection` drops every event whose version is not above the recorded version for its stream and skips the write when none remain, so a live update and a backfill page never fold one event twice, and the live path never meets a non-active generation of a history view because its building generation exists only while its source scopes are closed (E-9, D9, F1) -- [extension] The same projection is bound once per source stream type on the command declaration; the pipeline's step 9 feeds it each matching entry's events, and the backfill feeds it each source's history, so a cross-stream row has one projection and several bindings (E-9, E-7, D8) +- [extension] `applyAggregate` ignores the mode: the marker's `sourceVersions` decide, a marker that is not older than the input makes the write `skipped-newer`, and otherwise the marker's recorded contribution leaves the row of the marker's `aggregateKey` and the new contribution joins the row of `aggregateKeyOf`, one row moved by the difference when the key is the same and two rows when it changed, a row whose `entityCount` reaches 0 being deleted, and a `null` contribution joins no row and removes the marker; a live update on an entity backfill has not reached is therefore not partial, because `contribute` sees the whole DTO, which is why the missing-row rule of the per-entity path does not apply (E-9, D9, F1) +- [extension] An aggregate's entity is its source stream: `applyAggregate` derives the marker's `entityKey` from the input's one `StreamVersion` as `JSON.stringify([contextId, streamType, streamId])`, so the live path and the backfill, which pass `entry.version` and `dto.version`, name one marker, and equal stream IDs in two contexts or two stream types name two (E-9, D9, D2) +- [extension] A history projection folds events into a row: `keyOf` maps each event to the row it belongs to or to none, and `project` folds the events of one key, each stream's in stream-version order and the streams interleaved in any order, into the prior row; it is deterministic in the prior row and the events, reads nothing else, and is the one form a read model that depends on history across several streams may take, which is why such a read model rebuilds only under the write pause (E-9, D8, D9, Law 10) +- [extension] A history projection's fold gives the same row whatever the order in which the events of different streams arrive and however they are split between calls: from one prior row, the events of one key folded in one call or in several, with the streams interleaved in any way that keeps each stream's events in version order, give equal rows; D2 orders events only within a stream, and the live path feeds a row command by command while the backfill feeds it stream by stream, so a view whose meaning depends on the order across streams is not a history projection and waits for the ordered consumer of D18 (E-9, D2, D8, D18) +- [extension] `project` never answers `null`: a history row, once written, keeps its subject's ended state, such as a cancelled order, and leaves its table only when its generation is purged, so its `sourceVersions`, the fold's one memory, outlive the subject's end, and an event that arrives after it, such as `AllocationReleased` after `OrderCancelled`, folds into the row and never makes a new one (E-9, D9) +- [extension] `applyHistoryProjection` drops every event of the reserved type `baseline` before `keyOf`, on both paths: a baseline holds a stream's migrated state and no fact a view folds, and a migration that changes what a source's earlier events mean to the view is a new projection version and so a new generation, never a rewrite of rows in place (E-9, E-25, D9) +- [extension] A history row's `sourceVersions` hold, per source stream, the highest event version folded into it; `applyHistoryProjection` drops every event whose version is not above the recorded version for its stream and skips the write when none remain, so a live update and a backfill page never fold one event twice, and the live path never meets a non-active generation of a history view, because its generation in flight exists only while its source scopes are closed and `rollbackGeneration` of `spec:application.rebuild` refuses to reopen a retired one (E-9, D9, F1) +- [extension] A history projection is bound once per source stream type on the command declaration; the pipeline's step 9 feeds it, once per command, the events of every `streams` entry its bindings match, and the backfill feeds it each source's history, so a history row has one projection and several bindings; a `Projection` or an `AggregateProjection` takes one entry's DTO, so a read model of those forms derives each row from one source stream (E-9, E-7, D8) - A projection's version changes whenever its output for any input changes; a new version is added at the head of the read model's `projections` and is a new generation, never an in-place rewrite, and an earlier version leaves the list only when no generation in `building`, `verifying`, `verified` or `active` records it and none in `retired` is to be rolled back to (D8, D9) - The write rule reads and writes inside one mutation, so two writers of the same row serialize under optimistic concurrency and neither overwrites newer data (F1, D9) @@ -69,18 +76,23 @@ The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape w - typeWritableGeneration: `type WritableGeneration = { role: "active" | "building" | "verifying"; generation: number; projectionVersion: number }`, one generation a command writes, as the registry's `generationsToWrite` returns it; the role `verifying` covers the states `verifying` and `verified` (E-8, E-9, D9) - typeRowWriter: `type RowWriter = { db: GenericDatabaseWriter }`, the part of a ctx `applyProjection` uses, generic over the composition's data model because a database writer is not covariant in its data model; `RowDataModel` is the data model of one table that spreads `rowConventions` and has `by_key`, and the library reads and writes the read model's own table, named by its `table`, under that table's type (E-9) - typeAggregateProjection: `interface AggregateProjection { name: string; version: number; table: string; rowBudgetBytes?: number; aggregateKeyOf(tenantId: string, dto: D): string; contribute(tenantId: string, dto: D): number | null }` where `null` means the entity no longer contributes, such as a deleted subject, and `rowBudgetBytes` bounds the aggregate row and the marker alike (E-9, D9, F13) -- typeHistoryProjection: `interface HistoryProjection { name: string; version: number; table: string; rowBudgetBytes?: number; keyOf(tenantId: string, event: EventEnvelope): string | null; project(tenantId: string, prior: Row | null, events: EventEnvelope[]): Row | null }` where `EventEnvelope` is the envelope of `spec:context.event-envelope`, `null` from `keyOf` means the event is not this view's, `events` are the events of one key not yet folded into `prior`, in version order within each stream, and `null` from `project` deletes the row; a history row is a bounded summary of history, never a list that grows with it, so `rowBudgetBytes` binds it like any row (E-9, D8, D9, F13) -- fnApplyHistoryProjection: `applyHistoryProjection(ctx: MutationCtx, projection: HistoryProjection, input: { tenantId: string; events: EventEnvelope[]; mode: ApplyMode; generations: readonly WritableGeneration[] }): Promise<{ key: string; results: ApplyResult[] }[]>` grouping the events by `keyOf`, and per key and generation reading the row by `by_key`, dropping every event whose `streamVersion` is not above the row's recorded version for that stream, returning `skipped-newer` when none remain, otherwise writing `project(tenantId, prior, remaining)` with `sourceVersions` raised to the highest version folded per stream, in the same mutation; the pipeline's step 9 calls it with one `streams` entry's `events` and the rebuild's history batch with one `history` page (E-9, D8, D9, F1) +- typeHistoryProjection: `interface HistoryProjection { name: string; version: number; table: string; rowBudgetBytes?: number; keyOf(tenantId: string, event: EventEnvelope): string | null; project(tenantId: string, prior: Row | null, events: EventEnvelope[]): Row }` where `EventEnvelope` is the envelope of `spec:context.event-envelope`, `null` from `keyOf` means the event is not this view's, `prior` is `null` before the key's first event, `events` are the events of one key not yet folded into `prior`, each stream's in version order and the streams interleaved in any order, `project` answers a row and never `null`, and `rowBudgetBytes` is 16 KiB when absent; a history row is a bounded summary of history, never a list that grows with it, so `rowBudgetBytes` binds it like any row (E-9, D8, D9, F13) +- fnApplyHistoryProjection: `applyHistoryProjection(ctx: MutationCtx, projection: HistoryProjection, input: { tenantId: string; events: EventEnvelope[]; mode: ApplyMode; generations: readonly WritableGeneration[] }): Promise<{ key: string; results: ApplyResult[] }[]>` dropping the events of type `baseline`, grouping the rest by `keyOf` in the order given and leaving out those whose key is `null`, and per key and generation reading the row by `by_key`, dropping every event whose `streamVersion` is not above the row's recorded version for the event's stream, named by the envelope's `tenantId`, `contextId`, `streamType` and `streamId`, answering `skipped-newer` when none remain, and otherwise writing `project(tenantId, prior, remaining)` with `sourceVersions` raised to the highest version folded per stream and sorted by `contextId`, then `streamType`, then `streamId`, strings compared by UTF-16 code units, so the row is one value whatever the order the streams arrived in, and answering `inserted` or `updated`, never `deleted`, in the same mutation; `mode` is read by nothing, and a generation given whose `projectionVersion` is not the projection's `version` makes it throw the plain `Error` of `projectionOf` before it reads a row, a technical failure (E-9, D8, D9, F1, Law 6) - typeApplyMode: `type ApplyMode = "live-created" | "live-updated" | "backfill"` (E-9, D9) - typeApplyResult: `type ApplyResult = "inserted" | "updated" | "deleted" | "skipped-missing" | "skipped-newer" | "unchanged"` (E-9, D9) - fnApplyProjection: `applyProjection>(ctx: RowWriter, readModel: ReadModel, input: { tenantId: string; dto: D; versions: StreamVersion[]; mode: ApplyMode; generations: readonly WritableGeneration[] }): Promise` writing one result per generation in the order given; the pipeline's step 9 calls it with `versions: [entry.version]` for a per-entity row and the rebuild's batch with `versions: [dto.version]` (E-9, D8, D9) -- callers: the two callers are the command pipeline's step 9, with the mode from the `streams` entry's `created` flag and the list `generationsToWrite` returned, passed unchanged, and the rebuild's backfill and verify batches in `backfill` mode for the one generation they build or verify; a use case body never calls it, and the same two callers are the only callers of `applyAggregate` and `applyHistoryProjection` (E-9, D8, D9, E-7) -- tableOrderAllocationHistory: `orderAllocationHistory: defineTable({ ...rowConventions, orderId: v.string(), lines: v.array(v.object({ stockItemId: v.string(), quantity: v.number(), allocatedAt: v.optional(v.number()), releasedAt: v.optional(v.number()) })) }).index("by_key", ["tenantId", "generation", "key"])` the history-dependent cross-stream view of the example domain, keyed by order ID and folded from `OrderPlaced` in Orders and `StockAllocated` and `AllocationReleased` in Inventory, whose row is bounded by the order's line count and whose `sourceVersions` hold the order stream and every stock stream folded; it is bound once per source stream type and rebuilds only under the write pause, and Sc L2-6 builds it (E-9, E-46, D9, Sc L2-6) -- fnApplyAggregate: `applyAggregate(ctx: MutationCtx, projection: AggregateProjection, input: { tenantId: string; entityKey: string; dto: D; versions: StreamVersion[]; mode: ApplyMode; generations: readonly WritableGeneration[] }): Promise` writing one result per generation, `inserted` when the aggregate row was created, `updated` when `value` or `entityCount` changed, `skipped-newer` when the marker was not older, and `unchanged` when nothing moved; `mode` is accepted for symmetry with `applyProjection` and read by nothing (E-9, D9) -- aggregateStep1: for each generation it reads the marker by `by_entity` with the tenant, read model, generation and `entityKey`, and computes `contribution` as `contribute(tenantId, dto)` (E-9, D9) -- aggregateStep2: when a marker exists and its `sourceVersions` are not older than `input.versions` for the same stream, the result is `skipped-newer` and nothing is written; otherwise the difference is `contribution` minus the marker's recorded contribution, or minus zero when the marker is missing (E-9, D9, F1) -- aggregateStep3: it reads the aggregate row by `by_key` with `aggregateKeyOf(tenantId, dto)`, inserts it with `value` equal to the difference and `entityCount` 1 when missing, else adds the difference to `value` and adjusts `entityCount` by one for a new marker and by minus one for a removed one; then it inserts or replaces the marker with `contribution`, `aggregateKey` and `input.versions`, or deletes it when `contribution` is `null`, all in the same mutation (E-9, D9, E-8) -- validatorRowConventions: `const rowConventions = { tenantId: v.string(), generation: v.number(), key: v.string(), projectionVersion: v.number(), sourceVersions: v.array(streamVersionValidator) }` spread into every per-entity and cross-context read-model `defineTable` (E-9, Law 11) +- callers: the two callers are the command pipeline's step 9, with the mode from the `streams` entry's `created` flag and the list `generationsToWrite` returned, passed unchanged, and the rebuild's backfill and verify batches in `backfill` mode for the one generation they build or verify; a use case body never calls it, and the same two callers are the only callers of `applyAggregate` and `applyHistoryProjection`; step 9 calls `applyHistoryProjection` once per history read model per command, in the live-updated mode, with the events of every `streams` entry any of its bindings matches in the order of the entries, and a history batch calls it once with its one `history` page (E-9, D8, D9, E-7) +- tableOrderAllocationHistory: `orderAllocationHistory: defineTable({ ...rowConventions, orderId: v.string(), placedAt: v.optional(v.number()), cancelledAt: v.optional(v.number()), allocations: v.array(v.object({ stockItemId: v.string(), quantity: v.number(), allocatedAt: v.number(), releasedAt: v.optional(v.number()) })) }).index("by_key", ["tenantId", "generation", "key"])` the history-dependent cross-stream view of the example domain, the `HistoryProjection` named `orderAllocationHistory` over the table of that name, keyed by order ID and folded from `OrderPlaced` and `OrderCancelled` in Orders and `StockAllocated` and `AllocationReleased` in Inventory, with one entry of `allocations` per stock item the order allocated on and `sourceVersions` holding the order stream and every stock stream folded, so a row holds at most 100 entries and 101 versions; `getConvexSize` measures the largest such row, at 256-byte tenant and order IDs and 100 stock item IDs of 64 bytes, every allocation released, at 55,803 bytes, so the view declares `rowBudgetBytes` 65,536, the ceiling; it is bound once per source stream type, rebuilds only under the write pause, and Sc L2-6 builds it (E-9, E-46, D9, F13, Sc L2-6) +- typeOrderAllocationHistoryFields: `type OrderAllocationHistoryFields = { orderId: string; placedAt?: number; cancelledAt?: number; allocations: OrderAllocation[] }` where `type OrderAllocation = { stockItemId: string; quantity: number; allocatedAt: number; releasedAt?: number }`, the projected fields of the order allocation history, in `example/domain/orderAllocationHistory.ts` (E-9, E-46, Sc L2-6) +- fnOrderAllocationHistoryKeyOf: `orderAllocationHistoryKeyOf(tenantId: string, event: HistoryEvent): string | null` in `example/domain/orderAllocationHistory.ts`, answering the stream ID of an `OrderPlaced` or an `OrderCancelled`, the payload's `orderId` of a `StockAllocated` or an `AllocationReleased`, and `null` for every other event, where `type HistoryEvent = { streamId: string; eventType: string; recordedAt: number; occurredAt?: number; payload: unknown }` is the part of `EventEnvelope` the rules read, so the example's pure domain code imports no context library (E-9, E-46, Sc L2-6) +- fnProjectOrderAllocationHistory: `projectOrderAllocationHistory(tenantId: string, prior: OrderAllocationHistoryFields | null, events: readonly HistoryEvent[]): OrderAllocationHistoryFields` in the same file, folding the events one by one into `prior`, or, when `prior` is `null`, into `{ orderId, allocations: [] }` with the key of the first event that has one, and throwing the plain `Error` `"The order allocation history needs a prior row or an event of an order"` when there is neither; the view's `HistoryProjection` holds this function and `orderAllocationHistoryKeyOf` as its `project` and `keyOf` (E-9, E-46, Sc L2-6) +- orderAllocationHistoryFold: each field is written by the events of one stream, each event's time being its `occurredAt`, or its `recordedAt` when absent: `OrderPlaced` sets `placedAt` and `OrderCancelled` sets `cancelledAt`; `StockAllocated` on a stock item sets that stock item's entry of `allocations` to the payload's `quantity` and the time as `allocatedAt`, with no `releasedAt`; `AllocationReleased` sets `releasedAt` on that entry and changes nothing when the order has none on that stock item; every other event changes nothing; `allocations` is kept sorted by stock item ID, compared by UTF-16 code units; so the fold gives one row whatever the order across streams, a stock event before `OrderPlaced` included, and a pure test folds one order's events in every interleaving of its three streams and every split into calls and finds one row (E-9, D2, Sc L2-6) +- fnApplyAggregate: `applyAggregate(ctx: MutationCtx, projection: AggregateProjection, input: { tenantId: string; dto: D; versions: StreamVersion[]; mode: ApplyMode; generations: readonly WritableGeneration[] }): Promise` answering, per generation in order, one result for each aggregate row it writes, the old key's row before the new key's on a key change, `inserted` for a row it created, `updated` for a row whose `value` or `entityCount` changed and `deleted` for a row whose `entityCount` reached 0, or one `skipped-newer` when the marker was not older and one `unchanged` when no row moved; `input.versions` holds the one `StreamVersion` of the entity's source stream, and another length throws a plain error; `mode` is accepted for symmetry with `applyProjection` and read by nothing (E-9, D9) +- aggregateStep1: for each generation it derives `entityKey` from `input.versions[0]` as the contract states, reads the marker by `by_entity` with the tenant, read model, generation and `entityKey`, and computes `contribution` as `contribute(tenantId, dto)` and, when that is not `null`, `aggregateKey` as `aggregateKeyOf(tenantId, dto)` (E-9, D9) +- aggregateStep2: when a marker exists and its `sourceVersions` are not older than `input.versions` for the same stream, the result is `skipped-newer` and nothing is written (E-9, D9, F1) +- aggregateStep3: when a marker exists and `contribution` is `null` or `aggregateKey` differs from the marker's, it reads the row of the marker's `aggregateKey` by `by_key`, throwing a plain error when it is missing, a defect, subtracts the marker's contribution from `value` and 1 from `entityCount`, and deletes the row when `entityCount` reaches 0 and replaces it otherwise; when `contribution` is not `null` and the marker is missing or its key differs, it reads the row of `aggregateKey`, inserts it with `value` equal to `contribution` and `entityCount` 1 when missing, and otherwise adds `contribution` to `value` and 1 to `entityCount`; when the marker exists with the same key and `contribution` is not `null`, it adds `contribution` minus the marker's to that row's `value`, leaves `entityCount`, and writes no row when the difference is 0; when the marker is missing and `contribution` is `null`, it reads and writes no row (E-9, D9) +- aggregateStep4: then it inserts or replaces the marker with `entityKey`, `contribution`, `aggregateKey` and `input.versions`, or deletes it when `contribution` is `null`, all in the same mutation; a key change reads and writes two aggregate rows and the marker in each generation, and its two rows count against `limitReadModelWritesPerCommand` of `spec:application.read-models` (E-9, D9, E-8, E-45) +- validatorRowConventions: `const rowConventions = { tenantId: v.string(), generation: v.number(), key: v.string(), projectionVersion: v.number(), sourceVersions: v.array(streamVersionValidator) }` spread into every per-entity and history read-model `defineTable` (E-9, Law 11) - validatorAggregateRowConventions: `const aggregateRowConventions = { tenantId: v.string(), generation: v.number(), key: v.string(), projectionVersion: v.number(), value: v.number(), entityCount: v.number() }` spread into every aggregate read-model `defineTable`, with the same `by_key` index, where `key` is `aggregateKeyOf(tenantId, dto)` (E-9, Law 11, D9) - tableOrderTotals: `orderTotals: defineTable({ ...aggregateRowConventions }).index("by_key", ["tenantId", "generation", "key"])` an aggregate of order totals per status, where `key` is the status, `value` the sum of the orders' totals and `entityCount` the number of orders in it, given as the shape of an aggregate table and not as a read model the first experiment builds (E-9, D9) - validatorOrderSummaryFields: `const orderSummaryFields = { orderId: v.string(), status: v.union(v.literal("placed"), v.literal("cancelled")), lineCount: v.number(), total: v.number(), placedAt: v.number() }` the projected fields of the order summary, which its table spreads and its query's view validator spreads (E-9, E-46, First experiment) @@ -92,15 +104,15 @@ The shapes are TypeScript in the parent. `StreamVersion` is the kernel's shape w - step3: it decides by the mode and the generation's role: in the two live modes it inserts a missing row and replaces an existing one, except that live-updated in the role `building` answers `skipped-missing` for a missing row; in the backfill mode it inserts when missing and updates when the existing `sourceVersions` are older, else `skipped-newer`, whatever the role (D9, E-8, E-9) - step4: before a write it throws a plain error when the row measures above `rowBudgetBytes` with `getConvexSize`; then it writes with `ctx.db.insert`, `ctx.db.replace` or `ctx.db.delete` in the same mutation, a written row carrying the generation's `projectionVersion`, which is the version of the projection that built it, and returns the results in generation order (E-9, F1, F13) - limitRowSize: a read-model row stays under its read model's `rowBudgetBytes`, which `applyProjection` checks at every write, 16 KiB by default and at most 64 KiB, the library constant `limitRowBudgetBytes`, well under the 1 MiB document limit; the rebuild's batch bound is derived from the declared budget, and a row that would grow without bound with history is a wrong read model for this contract, a history projection included, whose row summarizes history within the same budget (F13, E-9, E-40) -- limitHistoryFold: on the live path `applyHistoryProjection` reads and writes at most one row per distinct key among one entry's events, each at most `rowBudgetBytes`, so a command that appends `n` events touches at most `n` history rows; on the backfill it takes one `history` page bounded by the rebuild's `limitHistoryBatch`, so one batch reads at most that many events plus as many rows (F13, E-9, E-40) +- limitHistoryFold: on the live path step 9 calls `applyHistoryProjection` once per history read model per command, so it reads and writes at most one row per distinct key among the command's events in each generation, each at most `rowBudgetBytes`, and a `PlaceOrder` of 100 lines, whose 101 `streams` entries key to one order, reads and writes the order allocation history's one row once; on the backfill it takes one `history` page of at most the progress row's `batchSize` events, as the rebuild's `limitHistoryBatch` bounds it, so one batch reads at most that many events plus as many rows (F13, E-9, E-40) - limitGenerationsPerWrite: at most 2 generations are written per row per command, the active one and the one in building, verifying or verified if any; a retired generation is kept but never written, so one read model costs at most 2 point reads and 2 writes per command and exactly 1 of each outside a rebuild (E-9, E-8, F13) -- determinismCheck: the acceptance suite folds a sample of DTOs through `project` twice and compares, and compares a backfilled row with the live-written row for the same entity (D8, D9, Sc L2-5) +- determinismCheck: the acceptance suite folds a sample of DTOs through `project` twice and compares, and compares a backfilled row with the live-written row for the same entity; a pure test folds one order's events through the order allocation history's `project` in every interleaving of its streams and every split into calls and finds one row; a test of `applyAggregate` moves an entity from one aggregate key to another and finds the old key's row without its contribution and count and the new key's with them (D8, D9, Sc L2-5, Sc L2-6) ## Verification — reviewed - A reviewer confirms that every read model in the corpus and the experiment is declared as a `ReadModel` with its `Projection`, an `AggregateProjection` or a `HistoryProjection` bound on a command declaration, and that the pipeline's step 9 and the rebuild's batches are the only callers of `applyProjection`, `applyAggregate` and `applyHistoryProjection`. -- A reviewer confirms that every `HistoryProjection` is bound with `pauseRequired` on its generations, that `applyHistoryProjection` drops already-folded events by the row's `sourceVersions` before it calls `project`, and that the L2-6 examples assert the fold against the journal. +- A reviewer confirms that every `HistoryProjection` is rebuilt with `pauseRequired` on its generations, that `applyHistoryProjection` drops baselines before `keyOf` and already-folded events by the row's `sourceVersions` before it calls `project`, that no history projection's `project` answers `null`, and that the L2-6 examples assert the fold against the journal. - A reviewer confirms that no `project` function touches `ctx`, the database, the scheduler or the environment. -- A reviewer confirms that every per-entity or cross-context read-model table spreads `rowConventions`, that every aggregate table spreads `aggregateRowConventions`, and that each carries the `by_key` index. -- A reviewer confirms that `applyAggregate` compares the marker's `sourceVersions` before it touches `value`, that it reads no mode, and that a `null` contribution removes the marker and its recorded contribution. +- A reviewer confirms that every per-entity or history read-model table spreads `rowConventions`, that every aggregate table spreads `aggregateRowConventions`, and that each carries the `by_key` index. +- A reviewer confirms that `applyAggregate` compares the marker's `sourceVersions` before it touches `value`, that it reads no mode, that it derives `entityKey` from the input's stream version, that a key change moves the contribution and the count from the marker's key's row to the new key's row, and that a `null` contribution removes the marker and its recorded contribution and creates no row. - A reviewer confirms that a `convex-test` test on the fixture composition writes one `CreateTwiceSummarizedDocument` into an active generation of `documentTitle` at version 1 and a generation being built at version 2 and finds on each row its own generation's `projectionVersion`, that a backfill batch of each generation over a document whose `title` is the depot's `deletedTitle` leaves version 1's row with that title and no row at version 2, the two versions' one difference, and that a test deletes a row through a `null` projection, `applyProjection` in the backfill mode at version 2 over that document, and finds the next live-updated write of the same document at a higher stream version with a live title, `applyProjection` called with the mode `live-updated` because no fixture command amends a document under `documentTitle`, answering `inserted` in the active generation and in a generation in `verifying`, and `skipped-missing` in one in `building`. diff --git a/design/specs/application/read-models.sdp.md b/design/specs/application/read-models.sdp.md index bab3e3f..3e8f240 100644 --- a/design/specs/application/read-models.sdp.md +++ b/design/specs/application/read-models.sdp.md @@ -29,7 +29,7 @@ relations: Layer 2 · Detail: full · Traces: D1, D7, D8, D9, D11, Law 5, Law 6, Law 9, Law 10, Law 11, F4, F7, F13, F15, Probe 3, Probe 5, S4, Sc L2-2, E-6, E-9, E-24, E-45. -A read model is a stored, queryable shape the parent maintains for an essential list, summary or small cross-context view. It is written inside the command that changes its sources, from the DTOs and stream versions the contexts return, so the core path runs zero projection jobs and an authorized query after a successful command sees the committed state or a later one. Where no read model is needed, the read is an authorized component query returning a DTO or an indexed query, and the simplest read that works wins. Reactive queries carry freshness to clients; nothing pushes events to sockets. +A read model is a stored, queryable shape the parent maintains for an essential list or summary, or for a view that folds the history of several streams; a small cross-context view of current state is a parent query over bounded component reads. It is written inside the command that changes its sources, from the DTOs and stream versions the contexts return, so the core path runs zero projection jobs and an authorized query after a successful command sees the committed state or a later one. Where no read model is needed, the read is an authorized component query returning a DTO or an indexed query, and the simplest read that works wins. Reactive queries carry freshness to clients; nothing pushes events to sockets. This Spec carries the read-need table as rules, the properties of a projection, the visibility promise and the CQRS reading the doc fixes. The projection signature shared by live update and rebuild, and the row conventions every read model follows, are pinned by `spec:application.projection-contract`. Rebuild of a read model is `spec:application.rebuild`. @@ -51,6 +51,7 @@ This Spec carries the read-need table as rules, the properties of a projection, - [non-blocking] Extension E-6, as it reaches queries: a parent query establishes and authorizes its actor through `authorizeQuery`, and its refusal is the command's rejection shape, `RejectionData` with the query's name in the field `entry` and the codes `unauthenticated` or `forbidden`; the owner confirmed the word, and nothing else of the shape is new (E-6, D7, D11, Law 5) - [non-blocking] Probe 3 pending: the cost of one component call in latency and function-call quota decides whether one entity's detail stays a component query or hot reads move, which is OQ1 (Probe 3, F4, OQ1) - [non-blocking] Extension E-45: the doc says a small cross-context view is a parent query over bounded component reads and does not give the bound; the option taken here is at most 8 component reads per parent query and at most 4 read-model rows written per command, both measured by the first experiment (E-45, D8) +- [non-blocking] Extension E-9, as it reaches read models: D8 lets a small cross-context view be a parent query over bounded component reads or a read model updated in the command; the option taken here is the parent query for a view of current state, because no form of `spec:application.projection-contract` writes one row from the DTOs of two sources, and the history projection for a view that folds the events of several streams; a composite row of current state is a form for the view that needs it (E-9, E-45, D8) ## Behavior @@ -59,6 +60,7 @@ This Spec carries the read-need table as rules, the properties of a projection, - rule: One entity's detail is read by an authorized component query returning a DTO (D8) - rule: An essential list or summary is read by an indexed query, or by a read model updated in the command (D8) - rule: A small cross-context view is read by a parent query over bounded component reads, or by a read model updated in the command (D8) +- rule: [extension] A small cross-context view of current state is a parent query over bounded component reads; no read-model form here writes one row from the DTOs of two sources, and a view that folds the events of several streams into one row is a history projection (E-9, E-45, D8) - rule: Broad reporting is outside the baseline (D8) - rule: A projection is named, versioned and deterministic (D8) - rule: A projection never issues commands or causes effects (D8, Law 10) @@ -80,13 +82,14 @@ This Spec carries the read-need table as rules, the properties of a projection, - flow: A client subscribes to a parent query with its tenant scope and the key of the entity, list or view it needs (D8, Law 11) - flow: The query authenticates and authorizes in the parent and returns nothing before authorization passes (Law 5, D11) - flow: For one entity's detail the query calls the context's `get` through `ctx.runQuery` on the component API with the tenant, and returns the DTO, which carries its stream version as the context's `toDto` maps it (D8, F11, F15) -- flow: For an essential list, summary or small cross-context view the query reads the active generation's rows of the read model by index and returns them with their source stream versions (D8, D9) +- flow: For an essential list or summary the query reads the active generation's rows of the read model by index and returns them with their source stream versions (D8, D9) +- flow: For a small cross-context view the query makes at most 8 component reads with the tenant and returns their DTOs as one `VersionedResult` with their stream versions (D8, E-45, F15) - flow: The client compares the source versions in the result with the versions its last command returned and knows whether it reads its own write or a later state (D8) - flow: When a later command changes a source, its mutation writes the row in the same transaction and every subscription on the query re-runs (D8, F7) ## Design -The read path has two shapes and the write path one. A parent `query` either reads read-model rows through an index that leads with the tenant, or calls context queries through `ctx.runQuery` with the tenant the parent authorized. The write path is the command pipeline's step 9, which calls `applyProjection` from the projection contract for every read model the declaration lists, from the `streams` entries the use case's executor returned, after its context calls and before the receipt. A query reads the read model's active generation from the generation registry in the same query, so a generation switch re-runs every subscription. +The read path has two shapes and the write path one. A parent `query` either reads read-model rows through an index that leads with the tenant, or calls context queries through `ctx.runQuery` with the tenant the parent authorized. The write path is the command pipeline's step 9, which calls `applyProjection` from the projection contract for each matching entry of every per-entity read model the declaration lists, and `applyHistoryProjection` once per history read model with the events of every matching entry, from the `streams` entries the use case's executor returned, after its context calls and before the receipt. A query reads the read model's active generation from the generation registry in the same query, so a generation switch re-runs every subscription. Nothing here schedules, and nothing here reads `ctx.auth` inside a component. The DTO shapes belong to the context that returns them; `StreamVersion` is the kernel's shape, which every context outcome carries, and `streamVersionValidator` is its validator. @@ -99,15 +102,15 @@ Nothing here schedules, and nothing here reads `ctx.auth` inside a component. Th - fnReadModelView: `readModelView(row: Row): Omit` what a read-model query returns for a row: the tenant, the key, the projection version, the source versions and the projected fields; the built-in page's rows carry `_id` and `_creationTime`, which an object validator of the row's fields refuses, and no document ID or generation number leaves the parent in a row; a page's cursors are the position tokens of `fnPageInGeneration`, which carry the generation and, as `paginator` issues them, the row's `_creationTime` and `_id`, and a client passes them back unread, since the list refuses a cursor that is not its own (D8, D2, E-9) - validatorReadModelView: `orderSummaryView = v.object({ tenantId: v.string(), key: v.string(), projectionVersion: v.number(), sourceVersions: v.array(streamVersionValidator), ...orderSummaryFields })` where `orderSummaryFields` are the projected fields `spec:application.projection-contract` declares for `tableOrderSummaries` (E-9, D8) - noActiveGeneration: a query over a read model that has no active generation throws a plain `Error` that names the read model, a technical failure, and never answers an empty page (D9, Law 6) -- fnCrossContextView: `export const getOrderWithStock = query({ args: { tenantId: v.string(), orderId: v.string() }, returns: v.union(v.object({ data: orderWithStockDto, sourceVersions: v.array(streamVersionValidator) }), v.null()), handler })` making at most 8 component reads and returning a `VersionedResult` whose `sourceVersions` are the DTOs' stream versions, or reading one cross-context read-model row, which carries `sourceVersions` itself (D8, E-45) -- typeVersionedResult: `[extension] type VersionedResult = { data: T; sourceVersions: StreamVersion[] }`, the result of a cross-context view composed from component reads, whose `data` is not a row; every query result carries source versions in one of three ways, a per-entity or cross-context read-model row carries `sourceVersions` by the row conventions so a list page needs no wrapper, an entity DTO carries its stream version from `toDto`, and a composed view is wrapped in this type, so the client can always compare with the versions its command returned; an aggregate row carries no comparable version, so a client that needs its own write reflected compares its entity's row, and the sum's freshness is the subscription's re-run when the command commits (E-45, D8, E-9) -- readModelRowShape: every read-model row carries `tenantId`, `generation`, `key`, `projectionVersion` and the projected fields, a per-entity or cross-context row carries `sourceVersions`, and an aggregate row carries `value` and `entityCount` with the versions on its markers, as the projection contract pins (E-9, D8, D9) +- fnCrossContextView: `export const getOrderWithStock = query({ args: { tenantId: v.string(), orderId: v.string() }, returns: v.union(v.object({ data: orderWithStockDto, sourceVersions: v.array(streamVersionValidator) }), v.null()), handler })` making at most 8 component reads and returning a `VersionedResult` whose `sourceVersions` are the DTOs' stream versions; no read-model row serves it, because no form writes one row from the DTOs of two contexts (D8, E-45, E-9) +- typeVersionedResult: `[extension] type VersionedResult = { data: T; sourceVersions: StreamVersion[] }`, the result of a cross-context view composed from component reads, whose `data` is not a row; every query result carries source versions in one of three ways, a per-entity or history read-model row carries `sourceVersions` by the row conventions so a list page needs no wrapper, an entity DTO carries its stream version from `toDto`, and a composed view is wrapped in this type, so the client can always compare with the versions its command returned; an aggregate row carries no comparable version, so a client that needs its own write reflected compares its entity's row, and the sum's freshness is the subscription's re-run when the command commits (E-45, D8, E-9) +- readModelRowShape: every read-model row carries `tenantId`, `generation`, `key`, `projectionVersion` and the projected fields, a per-entity or history row carries `sourceVersions`, and an aggregate row carries `value` and `entityCount` with the versions on its markers, as the projection contract pins (E-9, D8, D9) - indexReadModelByKey: `.index("by_key", ["tenantId", "generation", "key"])` on every read-model table (E-9, Law 11) - indexReadModelByKeyUse: point read of one row by the parent query and by `applyProjection` before an update; the generation comes from the registry's active row, which is one per read model and the same for every tenant, so a query narrows by its own `tenantId` and never by a tenant's generation (D8, D9, E-8, Law 11) - pagination: every list returns through `paginationResultValidator(item)` of `convex/server` and passes its one `paginate` call the options of `boundedPage`, as `spec:context.queries` pins; a list over a parent read-model table calls `paginator` of `convex-helpers` with the composition's schema, as a context list does, over the index of `indexReadModelList`, with the client's cursors moved into the active generation by `pageInGeneration`, relays `pageStatus` and `splitCursor` as the helper returns them, and maps the page's rows through `readModelView`; a list served by a context relays the context's page; a client pages either by an explicit cursor pair; a function runs at most one built-in `.paginate`, because Convex refuses a second with "Convex only supports a single paginated query in each function" (D8, S4, Probe 5, F15, E-24) - limitReadModelList: `limitReadModelList(readModel) = { items: limitListPage(readModel.rowBudgetBytes), bytes: limitListBytes }`, a function of the read model, so 100 items at the default 16 KiB and 64 at the 64 KiB cap; `paginator` honours the item cap on a first run and the row cap and the byte cap on every run, and answers `SplitRequired` with a `splitCursor` for a page that reaches either cap (F13, E-9, Probe 5) - limitComponentReadsPerQuery: `[extension]` 8 component reads per parent query (E-45, D8) -- limitReadModelWritesPerCommand: `[extension]` 4 read-model rows per command, counted as the rows step 9 inserts, replaces or deletes, a row in each generation once, and checked after each `streams` entry is applied; above it step 9 throws a plain error and nothing of the command commits (E-45, D8, Sc L2-3) +- limitReadModelWritesPerCommand: `[extension]` 4 read-model rows per command, counted as the rows step 9 inserts, replaces or deletes, a row in each generation once, and checked after each call of a write helper, one per `streams` entry for a per-entity read model and one per command for a history read model; an aggregate key change counts the two aggregate rows it writes, and a history row counts once however many entries fed it; above it step 9 throws a plain error and nothing of the command commits (E-45, D8, Sc L2-3) - readModelVisibility: a subscription re-runs when a row it read changes, including on generation switch, because the query reads the registry row too; after the switch and after a switch back each page shows the rows of the generation then active from the position its cursors name, so a client sees the same orders and never `InvalidCursor` (D8, F7, D9) - noProjectionJob: no scheduled function, cron, action or component maintains a read model on the core path; the only batch writer of read models is the rebuild (D8, D9) diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 7427e09..80f8dbd 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -46,6 +46,7 @@ The registry, the progress row, the per-entity marker and the gate document are - value: The common rebuild costs no downtime and no projection job; the write pause is paid only where a consistent cut is needed (D9) - risk: The standing cost is a generation registry read in every command that maintains a read model, a generation on every row, the earlier projection version kept in the code while its generation can be rolled back to, the batch chain, and for the write pause a gate every writer reads (D9) - risk: A pass costs at least one batch per tenant of the tenant list, because a batch names one tenant, so a deployment of many empty tenants pays one scheduled mutation for each (E-49, Law 11) +- risk: A paused rebuild closes its view's sources in every tenant until the switch or the abort; its backfill and its verify pass each take one batch per source stream, one more for each further `history` page of a stream longer than the batch size, and one per tenant and source to end that source's enumeration, so the pause lasts about twice that count times one batch; a batch that took streams until its event bound was spent, or a generation of one tenant, which the registry does not offer, would shorten it (D9, E-8, E-40, D18) - risk: The batch bound is a design choice under the transaction limits and no question of correctness: a batch that races a live command on one row is ordered by optimistic concurrency and the row ends at the source's newest stream version in either commit order, and a batch that reads past the read limit fails as a whole and leaves its checkpoint as it was (F13, F1, F17, Probe 6) - assumption: Mutations are serializable under optimistic concurrency, so a backfill batch and a live command that touch the same row serialize and the loser re-runs (F1) - assumption: A scheduled mutation that loses an optimistic-concurrency race is run again by Convex until it commits; the pinned packages do not show a bound on those runs, the design needs none for correctness, and Probe 6 shows what a stalled batch looks like (F9, F1, Probe 6) @@ -58,7 +59,7 @@ The registry, the progress row, the per-entity marker and the gate document are - [non-blocking] `@convex-dev/migrations` walks only the tables of the component it is defined in, reaches a context's rows only through a parent internal mutation written by hand against a contract its README does not document and the compiled tier needs a cast for, keeps a status apart from the generation's state, and leaves no error on its row when a batch fails past the read limit, so every batch is driven by the rebuild's own self-scheduled internal mutation and the progress row is the one checkpoint; a hosted deployment and sustained contention are not observed (Probe 6, F1, F17, D9) - [non-blocking] Extension E-40: the doc binds resumable batching to `@convex-dev/migrations` and gives no batch size; the component iterates a parent table while the sources of a read model live in a context component. The option taken here: a batch is one mutation with one cursor on the progress row and a bound of at most 100 entities, capped by the read model's declared `rowBudgetBytes` so that the parent's point reads and writes stay under 4 MiB, and by the context's byte-derived `list` page bound, which is 16 items at the default 256 KiB stream budget; every batch, the backfill and the verify pass over a context's enumeration and the purge over a parent table, is driven by the platform's own self-scheduled internal mutation, and the rebuild uses the component for nothing, because Probe 6 showed that it walks only the tables of the component it is defined in and drives a context's rows only through a parent internal mutation written by hand against a contract its README does not document; the progress row is the one checkpoint (E-40, E-24, E-9, F17, F13, F1) - [non-blocking] Extension E-8: the registry states, the rollback period, what a rollback does, the gate document, the scope a paused rebuild closes and the reach of a generation are design choices the doc does not make; they are pinned in the two contracts under this workflow (E-8, E-9, E-38, D9) -- [non-blocking] Extension E-8, what this workflow does not offer: the online path is pinned for a per-entity read model with one source stream type; an `AggregateProjection` and a row built from more than one source stream type have no rebuild here, and each gets one when a view needs it; the paused path of a history view is given below in outline, its cursor across several sources and its verify pass are that view's own rules, and `rollbackGeneration` refuses a generation that needs a write pause, because nothing would hold its sources while live commands fold onto a row that missed the events since its retirement (E-8, E-9, D9, D18) +- [non-blocking] Extension E-8, what this workflow does not offer: the online path is pinned for a per-entity read model with one source stream type and the paused path for a history view, a `HistoryRebuildTarget` with its sources; an `AggregateProjection` has no rebuild here and gets one when a view needs it, and a row made from the current state of more than one source has no form at all, as `spec:application.projection-contract` records; `rollbackGeneration` refuses a generation that needs a write pause, because nothing would hold its sources while live commands fold onto a row that missed the events since its retirement, so such a view returns to earlier rules only through a new generation under the pause (E-8, E-9, D9, D18) - [non-blocking] Extension E-49, a deployment whose grants predate the tenant list: the tenant list is written by `insertGrant`, so on such a deployment it holds no row while `grants` holds many, and a pass over an empty list visits no tenant and still ends; the option taken here is that `startGeneration` refuses an empty list beside a grant with a message naming `fillTenants`, and `fillTenants`, an operator entry, walks the grants in bounded batches through a progress row of its own and inserts each missing tenant once; a list that holds a tenant whose first grant came after the table and misses an older one is not detected by that refusal, so the operator runs `fillTenants` on such a deployment before its first `startGeneration` (E-49, E-48, E-40, Law 11) ## Workflow @@ -71,10 +72,11 @@ The registry, the progress row, the per-entity marker and the gate document are - rule: A backfill batch reads each source at its current stream version and writes the row only if the target is missing or older (D9) - rule: [extension] Coverage is verified before the switch by one verify pass over every tenant and every source, which inserts a missing row, updates an older one and deletes the row of a subject whose DTO projects to `null`; the generation is `verified` when that pass ends, whatever it repaired, and the count of repairs is kept for the operator (E-8, D9, Sc L2-5) - rule: [extension] The active generation switches in one write: one mutation that patches two generation rows together, the verified one to `active` and the active one to `retired`, or one row when the read model had no active generation, so no command and no query sees one row changed without the other; the old generation is kept for a rollback period (D9, E-8, F1) -- rule: [extension] A retired generation is not written; a rollback reopens the retired generation the operator names as `verifying`, one verify pass repairs what the writes since its retirement changed, presence and absence alike, and the ordinary switch makes it active, so a rollback is a verify pass and then a switch of one write, never one write by itself, and the current generation serves until that switch (E-8, D9, Sc L2-5) +- rule: [extension] A retired generation is not written; a rollback reopens the retired generation the operator names as `verifying`, one verify pass repairs what the writes since its retirement changed, presence and absence alike, and the ordinary switch makes it active, so a rollback is a verify pass and then a switch of one write, never one write by itself, and the current generation serves until that switch; a generation that needs a write pause is never reopened, and its view returns to earlier rules through a new generation under the pause (E-8, D9, Sc L2-5) - rule: [extension] Each generation is written with the projection whose version its row records, by its batches and by live commands alike, and `startGeneration` and `rollbackGeneration` refuse a version the read model does not declare (E-9, E-8, D8) - rule: Under the write pause every writer in the scope reads the maintenance gate, background writers are fenced or drained, and an operator can resume or abort (D9) - rule: A refused write under the pause is a transient refusal, retryable and never a stored rejection (D4, D9) +- rule: [extension] Under the pause a history row's recorded version for a stream means every event of that stream with the row's key up to that version is folded: a stream's events reach the fold in version order, from its first event on the backfill and the verify pass and after its last folded version on the live path, a generation in flight is written by its own batches alone while its sources are closed, and no retired generation is reopened; so a verify pass that repeats the backfill and folds no event finds the generation covering every source (E-8, E-9, D9, F1) - rule: Every batch has a stated bound below the transaction limits, tested against the pinned Convex version; the limits are ceilings, not batch sizes (D19, F13) - rule: [extension] Every function that starts or changes a generation is an operator entry of `spec:command.actor-and-scope`: an internal function that records the stated operator on the generation row and reads no grant (E-48, D11, D19) - rule: [extension] A generation spans every tenant of the deployment: its batches take one tenant at a time from `nextTenant` of the tenant list, in tenant-ID order, and name that tenant on every context call, so no call runs with an absent tenant as a wildcard, and the one registry row per read model answers every tenant's queries (D11, Law 11, E-8, E-49) @@ -94,13 +96,13 @@ The registry, the progress row, the per-entity marker and the gate document are - A batch that throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, and a developer error or a timeout ends the chain; the operator's `interruptGeneration` ends a chain on purpose, and `resumeGeneration` schedules a batch from the saved cursor with a new fence, at the batch size on the progress row; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) - Each verify batch enumerates the sources again from the progress row's cursor, which the end of the backfill reset to `null`, the first tenant, and calls `applyProjection` in backfill mode for every DTO, so a missing row is inserted, an older one updated, and the row of a DTO that projects to `null` deleted; each such repair is counted as a miss; when the last tenant's last page is done the generation moves to `verified` and no batch is scheduled (D9, E-8, Sc L2-5) - `switchGeneration` sets the verified generation `active` and, when the read model has an active generation, sets that one `retired` with `retiredAt` and `retireAfter`, in one mutation; every query reads the active generation from the registry, so every subscription on the read model re-runs (D9, F7, F1) -- [extension] During the rollback period the retired generation's rows are kept and not written; `rollbackGeneration` reopens the retired generation it is given as `verifying` with a new fence and a `null` cursor and schedules a verify batch, live commands write it again from that commit, its one verify pass repairs every row that is missing, older or left behind by a deleted subject, and the ordinary `switchGeneration` makes it active and retires the generation that served meanwhile (E-8, D9, Sc L2-5) +- [extension] During the rollback period the retired generation's rows are kept and not written; `rollbackGeneration` reopens the retired generation it is given, one without `pauseRequired`, as `verifying` with a new fence and a `null` cursor and schedules a verify batch, live commands write it again from that commit, its one verify pass repairs every row that is missing, older or left behind by a deleted subject, and the ordinary `switchGeneration` makes it active and retires the generation that served meanwhile (E-8, D9, Sc L2-5) - `abortGeneration` marks a building, verifying or verified generation `aborted`, leaves the active generation untouched, and resumes every closed gate entry that names this generation; an aborted rollback leaves the reopened generation `aborted`, never `retired` again (D9, Sc L2-6, E-8) - `purgeGeneration` starts the purge of an aborted generation, or of a retired one once `retireAfter` has passed; each purge batch deletes at most the progress row's batch size, capped by `limitPurgeBatch`, of rows of the generation for one tenant by the `by_key` range, stays on that tenant while a full batch came back, moves to the next tenant of the tenant list otherwise, and marks the generation `purged` after the last tenant; nothing starts a purge by itself, and a purge whose chain ended, a restore among the causes, is continued by `resumeGeneration` from its progress row (D19, E-40, E-49) - For a history-dependent read model the generation row records `pauseRequired`, and `startGeneration` closes the maintenance gate for every source scope of the read model's bindings in the mutation that inserts the row, recording its own generation on each closed entry, so the gate is closed before the first batch is scheduled and this generation's batches are the one background writer allowed through (D9, E-8, E-38) - While the gate is closed for the sources, every use case whose declaration writes one of them and the journal's baseline driver refuse with `writePaused` before any write, whether or not they maintain the view; a batch scheduled before the close reads the closed gate when it starts and refuses, a batch running when the close commits loses the optimistic-concurrency race on the gate document and re-runs into the refusal, and a batch of a superseded chain refuses because its generation fence is stale; nothing needs to drain (D9, D4, E-8, E-38, F1) -- Backfill under the pause enumerates a consistent cut because no source changes: each history batch takes one stream from the enumeration, reads one `history` page of it from the saved event cursor, folds the page into the building generation through `applyHistoryProjection` in backfill mode, and saves the stream and the event cursor on the progress row, moving to the next stream when the page reports `isDone`; an interrupted batch resumes from its cursor with the gate still closed, and the gate never reopens by itself (D9, E-9, Sc L2-6) -- Verification runs under the pause: each verify batch enumerates the sources again and checks, per stream, that the row of its last event's key records the stream's current version in `sourceVersions`, backfilling the stream's history in place when it does not; `switchGeneration` resumes every closed entry that names the generation in the mutation that makes it active, so writes reopen only after verification (D9, Sc L2-6) +- Backfill under the pause enumerates a consistent cut because no source changes: each history batch takes the next stream of its cursor's source in its tenant, reads one `history` page of it from the saved event cursor, folds the page into the building generation through `applyHistoryProjection` in backfill mode, and saves the source, the stream and the event cursor on the progress row, moving to the next stream when the page reports `isDone`, to the next source when that source's enumeration in the tenant ends, and to the next tenant after the last source; an interrupted batch resumes from its cursor with the gate still closed, and the gate never reopens by itself (D9, E-9, Sc L2-6) +- Verification runs under the pause: the verify pass repeats the backfill's batches from the first tenant and source and counts as a miss every row they insert or update, which over a covered generation is none, so a miss is repaired by the batch that found it and the generation is `verified` when the pass ends; `switchGeneration` resumes every closed entry that names the generation in the mutation that makes it active, so writes reopen only after verification (D9, E-9, Sc L2-6) ## Design @@ -111,9 +113,11 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - transactionBoundary: each batch is one scheduled internal mutation that commits its rows, its checkpoint and the scheduling of the next batch together; every operator entry that changes anything is one mutation, and `getGenerations` is one query (D9, F8, F9) - convexSurface: `startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration`, `purgeGeneration` and `fillTenants` as the operator entries, `backfillBatch`, `verifyBatch`, `purgeBatch` and `fillTenantsBatch` as the scheduled batches, all internal mutations, and `getGenerations` as an internal query, in the composition's `rebuild` module; the gate entries are in the write-pause contract (D9, E-8, E-48) - typeRebuildTarget: `type RebuildTarget = { readModel: ReadModel; source: SourceRef; list: FunctionReference<"query", "internal", ListArgs, PaginationResult> }` one per-entity read model the deployment can rebuild, its one source stream type, and the reference to that stream type's `list` query in its context component, such as `components.orders.queries.order.list` (E-8, E-9, D10) -- typeRebuildConfig: `type RebuildConfig = { targets: readonly RebuildTarget[]; refs: { backfillBatch: BatchRef; verifyBatch: BatchRef; purgeBatch: BatchRef; fillTenantsBatch: FillBatchRef } }` where `type BatchRef = FunctionReference<"mutation", "internal", { generationId: Id<"generations">; fence: number }, null>`, `type FillBatchRef = FunctionReference<"mutation", "internal", { fence: number }, null>` and `refs` are the composition's own registered batches, which a batch schedules by reference (E-40, E-49, F8) -- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; the tenant list check, throwing `"The tenant list is empty while grants exist; fillTenants fills it from the grants table"` when `nextTenant(ctx, null)` answers `null` and `ctx.db.query("grants").first()` answers a row, and `"The tenant list is being filled; startGeneration waits until fillTenants has ended"` when the `tenantFill` row is in the pass `fill`; then it inserts the generation row with the next number, one above the last row of `by_read_model` in descending order and 1 when there is none, `state: "building"`, `pauseRequired: false`, since a `RebuildTarget` is a per-entity read model and the paused start of a history view is given only in outline, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `pass: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48, E-49) -- fnBatchSizeFor: `batchSizeFor(readModel: ReadModel, requested?: number): number` throwing `"The batch size must be a whole number of at least 1, and " + requested + " is not"` unless `requested` is absent or an integer of at least 1, and returning `min(requested ?? rebuildBatchCeiling(readModel), rebuildBatchCeiling(readModel))`, so a batch size of 0, a negative one and a fraction are refused and a larger one is lowered to the ceiling (E-40, F13) +- typeHistorySource: `type HistorySource = { source: SourceRef; list: FunctionReference<"query", "internal", ListArgs, PaginationResult<{ version: StreamVersion }>>; history: FunctionReference<"query", "internal", { tenantId: string; streamId: string; fromVersion?: number; paginationOpts: PaginationOptions }, PaginationResult> }` one source stream type of a history view, with the references to its `list` and `history` queries of `spec:context.queries` in its context component, such as `components.inventory.queries.stockItem.list` and `components.inventory.queries.stockItem.history` (E-8, E-9, D10) +- typeHistoryRebuildTarget: `type HistoryRebuildTarget = { readModel: HistoryProjection; sources: readonly [HistorySource, ...HistorySource[]] }` one history view the deployment can rebuild under the write pause, with every context and stream type pair its bindings name, in the order a pass takes them; the order allocation history's lists the stock item source first and the order source second (E-8, E-9, E-38, D9) +- typeRebuildConfig: `type RebuildConfig = { targets: readonly (RebuildTarget | HistoryRebuildTarget)[]; refs: { backfillBatch: BatchRef; verifyBatch: BatchRef; purgeBatch: BatchRef; fillTenantsBatch: FillBatchRef } }` where `type BatchRef = FunctionReference<"mutation", "internal", { generationId: Id<"generations">; fence: number }, null>`, `type FillBatchRef = FunctionReference<"mutation", "internal", { fence: number }, null>` and `refs` are the composition's own registered batches, which a batch schedules by reference; a target that carries `sources` is a history view, and no two targets share a read model name (E-40, E-49, F8) +- fnStartGeneration: `export const startGeneration = internalMutation({ args: { readModel: v.string(), projectionVersion: v.optional(v.number()), batchSize: v.optional(v.number()), operator: v.string() }, returns: v.id("generations"), handler })` in order: `assertOperator`; the target by name, else `"This deployment declares no read model " + readModel`; the version, `projectionVersion` when given and else the version of the first of the read model's `projections`, checked by `projectionOf`; the batch size by `batchSizeFor`; the in-flight check, throwing `"Read model " + name + " already has generation " + n + " in " + state` when a generation is `building`, `verifying` or `verified`; the tenant list check, throwing `"The tenant list is empty while grants exist; fillTenants fills it from the grants table"` when `nextTenant(ctx, null)` answers `null` and `ctx.db.query("grants").first()` answers a row, and `"The tenant list is being filled; startGeneration waits until fillTenants has ended"` when the `tenantFill` row is in the pass `fill`; then it inserts the generation row with the next number, one above the last row of `by_read_model` in descending order and 1 when there is none, `state: "building"`, `pauseRequired` false for a `RebuildTarget` and true for a `HistoryRebuildTarget`, whose start `historyStart` states, `fence: 1`, `startedAt` and `changedAt` at the mutation's `Date.now()` and `startedBy` and `changedBy` the operator, inserts the progress row with `pass: "backfill"`, the batch size, `cursor: null`, every count 0 and `updatedAt` at the same `Date.now()`, schedules `refs.backfillBatch` with fence 1 through `ctx.scheduler.runAfter(0, ...)`, and returns the generation row's ID (D9, E-8, E-9, E-40, E-48, E-49) +- fnBatchSizeFor: `batchSizeFor(readModel: ReadModel | HistoryProjection, requested?: number): number` throwing `"The batch size must be a whole number of at least 1, and " + requested + " is not"` unless `requested` is absent or an integer of at least 1, and returning `min(requested ?? rebuildBatchCeiling(readModel), rebuildBatchCeiling(readModel))`, so a batch size of 0, a negative one and a fraction are refused and a larger one is lowered to the ceiling (E-40, F13) - fnBackfillBatch: `export const backfillBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9, E-40) - fnVerifyBatch: `export const verifyBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9) - fnPurgeBatch: `export const purgeBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D19, E-40) @@ -143,17 +147,19 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - errorUnknownGeneration: every entry and every helper that takes a `generationId` throws `"No generation " + generationId` when the row is absent, and a batch returns without writing instead (E-8) - stallSignal: a batch that keeps losing an optimistic-concurrency race commits nothing and so cannot count its own losses; a stalled generation is one whose progress row's `updatedAt` has stopped advancing while its pass is not `idle`; `resumeGeneration` takes no batch size, so the operator answers a stalled backfill or verify pass with `abortGeneration` and a new `startGeneration` with a smaller `batchSize`, down to 1, and a purge batch deletes rows of a generation nothing else writes (E-40, F1, F9, Probe 6) - occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch; a batch reads the generation row and never writes it outside the end of a pass, and it writes the progress row, which no command reads, so a batch's checkpoint re-runs no command (F1, D9, Probe 6, E-40) -- limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) -- limitPurgeBatch: `limitPurgeBatch(readModel: ReadModel): number`, `min(500, floor(4194304 / readModel.rowBudgetBytes))` rows, the cap on the rows a purge batch deletes, which deletes the progress row's `batchSize` when that is smaller; 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) +- limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch of a per-entity read model and events per batch of a history view, a history projection that declares no `rowBudgetBytes` counting 16 KiB, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) +- limitPurgeBatch: `limitPurgeBatch(readModel: ReadModel | HistoryProjection): number`, `min(500, floor(4194304 / readModel.rowBudgetBytes))` rows, a history projection that declares no `rowBudgetBytes` counting 16 KiB, the cap on the rows a purge batch deletes, which deletes the progress row's `batchSize` when that is smaller; 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) - limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch returns without writing (E-40, F13) - limitBuildingGenerations: at most one generation in `building`, `verifying` or `verified` per read model at a time, so a command writes at most two generations, the active one and that one (E-8, E-9) - limitRollbackPeriod: `[extension]` 7 days by default, stored as `retireAfter` on the retired row; the retired generation is kept and not written, and a rollback at or after `retireAfter` is refused (E-8, D9) - limitGenerationsListed: 20 generations per `getGenerations` answer, the newest first, a library constant (E-8, F13) -- fnHistoryPageCall: `ctx.runQuery(components.orders.queries.order.history, { tenantId, streamId, paginationOpts: { numItems: limitHistoryBatch, cursor: eventCursor } })` the one read of context data a history batch makes beside the `list` call of `numItems` 1 that gives it its next stream, returning a `paginator` page of envelopes in version order that the batch folds through `applyHistoryProjection` (D2, D10, E-9, E-24) -- historyStep3: for a history view a batch replaces steps 4 and 5: when the cursor names no stream it calls `list` with `numItems` 1 from the page cursor to take the next stream and sets `eventCursor` to `null`; it then calls `history` for that stream with `limitHistoryBatch` events from `eventCursor`, calls `applyHistoryProjection` in backfill mode for the building generation with the page, and saves `eventCursor`, or clears `streamId` and advances the page cursor when the page reports `isDone` (D9, E-9, E-24, E-40) -- historyVerify: a verify batch of a history view enumerates the sources with `list` and, per stream, reads its last event through `history` with `fromVersion` at the stream's version and `numItems` 1, finds that event's row by `keyOf`, and counts a miss when the row is absent or its `sourceVersions` record a lower version for the stream; a miss is repaired by folding the stream's history in place through the same history batches from `eventCursor` null (D9, E-9, Sc L2-6) +- fnHistoryPageCall: `ctx.runQuery(source.history, { tenantId, streamId, paginationOpts: { numItems: batchSize, cursor: eventCursor } })` with `source` the entry of the target's `sources` the cursor names and `batchSize` the progress row's, the one `history` call a history batch makes beside its one `list` call of `numItems` 1, returning a `paginator` page of envelopes in version order, capped by the query's own item and byte bounds, that the batch folds through `applyHistoryProjection` (D2, D10, E-9, E-24) +- historyCursor: a history view's batch cursor adds `sourceIndex`, the position in the target's `sources`, so a pass goes tenant by tenant in tenant-ID order, source by source in `sources` order within a tenant, stream by stream in the source's `list` order and page by page in version order within a stream, and reads the stream a cursor names from that source's `history`, so two sources with equal local stream IDs never meet; a pass of a history view costs at least one batch per tenant and source, the batch whose `list` page holds no stream (E-8, E-40, E-49, Law 11) +- historyStart: for a `HistoryRebuildTarget`, `startGeneration` takes the version the projection declares and refuses a given `projectionVersion` that differs with the message of `projectionOf`, sets `pauseRequired: true`, and, after inserting the generation row and the progress row and before scheduling, calls the write pause's `closeScope` once per entry of `sources`, in order, with `"source:" + contextId + ":" + streamType`, the reason `"rebuild of " + name + " generation " + n`, the operator and the new generation's ID, so a scope already closed or a full gate refuses the whole start and no batch is scheduled (D9, E-8, E-38, E-48) +- historySteps: for a history view a backfill or verify batch keeps steps 1 to 3 and replaces steps 4 to 6: with `source` the entry of `sources` at the cursor's `sourceIndex`, 0 when the cursor is `null`, when the cursor names no stream it calls `source.list` with `numItems` 1, `includeDeleted` and the page cursor; a page with no DTO reads no history: when it is not done the batch checkpoints at its `continueCursor`, and when it is done that source ends in the tenant and the batch checkpoints at the next source of the same tenant with a `null` page cursor, or after the last source at `{ tenantId: next, pageCursor: null, sourceIndex: 0 }` for `next = nextTenant(ctx, tenantId)`, the pass being done when `next` is `null`; a page with a DTO names the stream, its `version.streamId`, with a `null` event cursor and the page's `continueCursor` as the page cursor; for the stream the cursor or the page names, the batch then calls `source.history` as `fnHistoryPageCall` states, calls `applyHistoryProjection(ctx, target.readModel, { tenantId, events: page, mode: "backfill", generations: [thisGeneration] })`, adds one to `rowsWritten`, or to `misses` in a verify batch, for each `inserted` or `updated` and one to `rowsSkipped` for each `skipped-newer`, and checkpoints with the stream and the page's `continueCursor` as the event cursor, or with neither when the page reports `isDone`; every checkpoint patches the progress row and schedules the next batch as step 6 does for a pass that is not done, and the batch whose pass is done moves the generation and the progress row as step 6 and `verifySteps` state (D9, E-9, E-24, E-40, E-49) +- historyVerify: a verify batch of a history view runs `historySteps` from the `null` cursor the end of the backfill set, with `state` `verifying` and `pass` `verify`; under the pause no source changes, so over a generation the backfill covered every key answers `skipped-newer`, an event of type `baseline` or whose key is `null` is never a miss, and a row the pass finds missing or behind is repaired by the batch that read the events; the generation is `verified` when the pass ends, as for every read model, and the pass reads every source's history once more under the pause (D9, E-8, E-9, Sc L2-6) - rebuildClass: a read model declares its class at declaration time, per entity for the online path, as a `ReadModel` with its projections, and history across streams for the write pause, as a `HistoryProjection`; `pauseRequired` on the generation row records it, and a history view never rebuilds online because a live fold onto a row backfill has not reached cannot be told apart from a complete one (D9, D18, E-9) -- limitHistoryBatch: [extension] `min(500, floor(4 MiB / (2 * rowBudgetBytes)))` events per history batch, 128 at the default 16 KiB row budget and 32 at the 64 KiB ceiling, so one batch reads at most that many events of at most `limitEventBytes` of `spec:context.event-envelope`, 20,480 bytes and the 61 bytes of a stored document's system fields, about 2.5 MiB, one stream row through `list`, and at most one history row per event, at most 2 MiB, and writes at most 2 MiB, under the ceilings with the 1 second timeout in mind; the count is stored on the progress row as its `batchSize` by `startGeneration` and never changed after, like the online bound (E-40, E-9, E-24, F13) +- limitHistoryBatch: [extension] a history batch reads at most the progress row's `batchSize` events, which `startGeneration` sets through `batchSizeFor` like every read model's and nothing changes after, so the one formula of `limitBatchSize` gives 100 events at the default 16 KiB row budget and 32 at the 64 KiB ceiling, the order allocation history's; one batch reads one stream row of at most the stream's `budgetBytes` through its one `list` call, those events through its one `history` call, each at most `limitEventBytes` of `spec:context.event-envelope` and the 61 bytes of a stored document's system fields, with the page's baselines held to about 9 MiB by the query's byte bound, and at most one history row per event, at most 2 MiB, and writes at most 2 MiB, under the ceilings with the 1 second timeout in mind (E-40, E-9, E-24, F13) ## Example space @@ -165,7 +171,7 @@ When the backfill is interrupted after {batches:number} batches and then {resump Then no row of the new generation holds data older than its source stream version And the new generation {coverage:"covers every stream enumerated at cutover"|"is discarded and read by no query"} And no command or external effect ran during the rebuild -And the active generation {cutover:"switches in one write and can roll back"|"stays the old one"} +And the active generation {cutover:"switches in one write and can roll back"|"switches in one write and refuses a rollback"|"stays the old one"} And writes {writes:"never stopped"|"reopen only after verification"|"reopen after the abort"} ``` @@ -178,6 +184,7 @@ And writes {writes:"never stopped"|"reopen only after verification"|"reopen afte - A reviewer confirms that every entry that changes a generation calls `assertOperator` first and writes the operator on the row, and that no rebuild function reads a grant or `ctx.auth`. - A reviewer confirms that the gate is read inside every writer's mutation, not cached, that every batch reads it through `gateAllows` and parks its generation instead of throwing, that `startGeneration` closes one entry per source of a paused view and nothing else, and that the write-pause examples assert a refusal of a source writer while closed, the fold of the history view against the journal, and a reopen only after verification. - A reviewer confirms that no registry function, batch or query takes a tenant scope for a generation, that every batch takes its tenant from `nextTenant` and names it on its context call, and that L2-5's example seeds more than one tenant and asserts both are covered at cutover. +- A reviewer confirms that a history target's `sources` name exactly the context and stream type pairs the view's bindings name, so `startGeneration` closes every writer of its sources, and that `rollbackGeneration` refuses every generation with `pauseRequired`. - A reviewer confirms that `resumeChain` is the only code path that reschedules a stranded generation and that the restore's `finishRestoreRun` calls it for every generation its inventory lists in `building` or `verifying`. - A reviewer confirms that `fillTenants` and `fillTenantsBatch` write only tenant rows and the `tenantFill` row, read a grant only through `by_principal` to find the next tenant, insert a tenant row only after a point read finds none, and that `startGeneration` refuses an empty tenant list beside a grant and a fill in its pass `fill`, and `fillTenants` a generation in flight. - A reviewer confirms that no function writes a retired generation, that `rollbackGeneration` is the only transition out of `retired` besides purge, and that L2-5's example asserts that the reopened generation is `verified`, holds a row for a subject created since its retirement and none for a subject deleted since, before it is active. diff --git a/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md b/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md index 19af9de..542f8ce 100644 --- a/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md +++ b/design/specs/application/rebuild.write-pause-rebuild-interrupt.sdp.md @@ -23,16 +23,18 @@ When the backfill is interrupted after {batches: 2} batches and then {resumption Then no row of the new generation holds data older than its source stream version And the new generation {coverage: "covers every stream enumerated at cutover"} And no command or external effect ran during the rebuild -And the active generation {cutover: "switches in one write and can roll back"} +And the active generation {cutover: "switches in one write and refuses a rollback"} And writes {writes: "reopen only after verification"} ``` ## Verification — executable - Runs in the native tier; every test owns its disposable backend. -- The view is the order allocation history of `spec:application.orders-inventory-example`, a `HistoryProjection` bound to the order stream type and the stock item stream type. +- The view is the order allocation history of `spec:application.orders-inventory-example`, a `HistoryProjection` bound to the order stream type and the stock item stream type, whose rebuild target lists the stock item source first, so the backfill folds every stock event of an order before its `OrderPlaced`. - The test starts a generation with `pauseRequired` and asserts that the gate holds one closed `source:` entry for each of the two stream types, each naming this generation. - The test asserts that `PlaceOrder` and `CancelOrder` are refused with `writePaused` because their declarations' `writes` name a closed source, that the error's `data.kind` is `"transient"`, and that nothing is stored. - The test interrupts after the second history batch, asserts the gate is still closed, resumes with `resumeGeneration`, and asserts the generation reaches `verified` with zero misses. -- The test asserts that every row of the new generation equals the fold of its order's events and its stock items' allocation events read from the two journals through `history`, and that its `sourceVersions` hold each source stream's current version. +- The test asserts that every row of the new generation equals the fold of its order's events and its stock items' allocation events read from the two journals through `history`, and that its `sourceVersions` hold, per stream, the version of that stream's last event with the order's key; the sample holds a cancelled order whose stock items were released after the cancel, a stock item received and never allocated, and a stock item that allocated to a second order after the first. +- The test asserts that each row of the new generation equals, beside `generation`, the row the old generation holds for the same order, which live commands wrote. - The test asserts that a `PlaceOrder` sent once the generation is `verified` and before the switch is still refused, then switches, which resumes both source scopes, and asserts that the same command now succeeds and that the active generation is the new one. +- The test calls `rollbackGeneration` on the generation the switch retired and asserts the refusal `"Generation " + n + " of orderAllocationHistory needs a write pause and cannot be rolled back"`, a gate with no closed entry, and the new generation still active. diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index c799b3a..f5a4d65 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -50,6 +50,7 @@ The write pause is the maintenance gate every writer in a scope reads inside its - `startGeneration` closes one entry per source of a paused view and records its own generation on each in the mutation that inserts the generation row, so every entry that names a generation names a row that exists (D9, E-8) - [extension] The operator entries are `closeGate` and `resumeGate` for the scope `all` and for a tenant scope, and `getGate` and `getGateAudit` to read; each follows the operator entry contract of `spec:command.actor-and-scope`, and a source scope is closed and reopened only by the rebuild (E-8, E-48, D19) - Abort marks the generation aborted and resumes its scopes in one mutation (D9, Sc L2-6) +- [extension] A source scope closes only for a generation `startGeneration` registers: `rollbackGeneration` of `spec:application.rebuild` refuses a retired generation with `pauseRequired`, because no function would close its sources while live commands fold onto rows that missed the events since its retirement, so a history view returns to earlier rules only through a new generation, whose start closes its sources again (E-8, D9, Sc L2-6) - The restore procedure cannot use the gate document, because the snapshot replaces it; the restore door keeps every writer out through the environment, which survives a restore because backups exclude environment variables; the restore procedure's own mutations, which write only `restoreRuns` and the generation rows the accepted branch resumes with their progress rows, are the one exemption, and that workflow states it (D19, F12, Sc L2-8, E-42) - [extension] A closed entry the snapshot brings back is as closed as one written live: an entry that names a generation stays closed while the restore's pending-work step resumes that generation, and reopens only when that generation is switched or an operator aborts it, and a tenant or `all` entry stays closed until an operator resumes it, so the gate never reopens by itself through a restore either (E-8, E-42, F12, D19) - [extension] Gate changes are treated as mandatory audit: every close and every resume inserts one record through `writeOperatorAudit` of `spec:operations.baseline-operations` inside the same mutation and fails closed, so a gate change whose audit insert fails changes nothing: the gate document is as it was and no record exists; the doc fails closed only for mandatory business or security audit and does not name gate changes among them (E-8, E-43, D19) diff --git a/example/domain/orderAllocationHistory.ts b/example/domain/orderAllocationHistory.ts new file mode 100644 index 0000000..49dc70e --- /dev/null +++ b/example/domain/orderAllocationHistory.ts @@ -0,0 +1,116 @@ +// The order allocation history's rules: keyOf maps an event to the order it belongs to, and project +// folds one order's events into its row (spec:application.projection-contract, +// orderAllocationHistoryFold). Each field of the row is written by the events of one stream, +// placedAt and cancelledAt by the order's and each allocation by its stock item's, and the +// allocations are kept in stock item ID order, so the row is one value whatever the order in which +// the events of different streams arrive. The parent's HistoryProjection holds these rules; no +// context mounts them, so index.ts does not export them. +import type { OrderEvent } from "./order.js"; +import type { StockItemEvent } from "./stockItem.js"; +// The part of an event envelope the rules read; the context library's EventEnvelope has each field. +export type HistoryEvent = { + streamId: string; + eventType: string; + recordedAt: number; + occurredAt?: number; + payload: unknown; +}; +// The quantity one stock item allocated to the order, when, and when it was released. +export type OrderAllocation = { + stockItemId: string; + quantity: number; + allocatedAt: number; + releasedAt?: number; +}; +export type OrderAllocationHistoryFields = { + orderId: string; + placedAt?: number; + cancelledAt?: number; + allocations: OrderAllocation[]; +}; +type OrderEventType = OrderEvent["eventType"]; +type AllocationPayload = Extract< + StockItemEvent, + { eventType: "StockAllocated" | "AllocationReleased" } +>["payload"]; +const orderEventTypes: readonly OrderEventType[] = [ + "OrderPlaced", + "OrderCancelled", +]; +// The order an event belongs to, or null for an event that is not this view's. +export function orderAllocationHistoryKeyOf( + _tenantId: string, + event: HistoryEvent, +): string | null { + if ((orderEventTypes as readonly string[]).includes(event.eventType)) + return event.streamId; + if ( + event.eventType === "StockAllocated" || + event.eventType === "AllocationReleased" + ) + return (event.payload as AllocationPayload).orderId; + return null; +} +// Stock item IDs compared by UTF-16 code units, the order the allocations are kept in. +const byStockItemId = (left: OrderAllocation, right: OrderAllocation) => + left.stockItemId < right.stockItemId + ? -1 + : left.stockItemId > right.stockItemId + ? 1 + : 0; +function foldOne( + row: OrderAllocationHistoryFields, + event: HistoryEvent, +): OrderAllocationHistoryFields { + const time = event.occurredAt ?? event.recordedAt; + switch (event.eventType) { + case "OrderPlaced": + return { ...row, placedAt: time }; + case "OrderCancelled": + return { ...row, cancelledAt: time }; + case "StockAllocated": { + const { quantity } = event.payload as AllocationPayload; + const allocations = row.allocations.filter( + (allocation) => allocation.stockItemId !== event.streamId, + ); + allocations.push({ + stockItemId: event.streamId, + quantity, + allocatedAt: time, + }); + return { ...row, allocations: allocations.sort(byStockItemId) }; + } + case "AllocationReleased": + return { + ...row, + allocations: row.allocations.map((allocation) => + allocation.stockItemId === event.streamId + ? { ...allocation, releasedAt: time } + : allocation, + ), + }; + default: + return row; + } +} +// Folds the events of one order, each stream's in version order, into its prior row, or into a new row +// named by the first event's key. A row is never deleted: a cancelled order keeps its row, so a +// release after the cancel folds into it. +export function projectOrderAllocationHistory( + tenantId: string, + prior: OrderAllocationHistoryFields | null, + events: readonly HistoryEvent[], +): OrderAllocationHistoryFields { + let start = prior; + if (start === null) { + const orderId = events + .map((event) => orderAllocationHistoryKeyOf(tenantId, event)) + .find((key) => key !== null); + if (orderId === undefined || orderId === null) + throw new Error( + "The order allocation history needs a prior row or an event of an order", + ); + start = { orderId, allocations: [] }; + } + return events.reduce(foldOne, start); +} diff --git a/tests/pure/order-allocation-history.test.ts b/tests/pure/order-allocation-history.test.ts new file mode 100644 index 0000000..1b99f0b --- /dev/null +++ b/tests/pure/order-allocation-history.test.ts @@ -0,0 +1,205 @@ +// The order allocation history's fold gives one row whatever the order in which the events of +// different streams arrive and however they are split between calls +// (spec:application.projection-contract, orderAllocationHistoryFold). +import { expect, test } from "vitest"; +import { + orderAllocationHistoryKeyOf, + projectOrderAllocationHistory, + type HistoryEvent, + type OrderAllocationHistoryFields, +} from "../../example/domain/orderAllocationHistory.js"; +import type { EventEnvelope } from "../../src/context/index.js"; +// The rules take the envelope the adapter's streams entry and the history query carry. +const fromEnvelope = (event: EventEnvelope): HistoryEvent => event; +const tenantId = "t-1"; +function envelope( + contextId: string, + streamType: string, + streamId: string, + streamVersion: number, + eventType: string, + payload: unknown, + recordedAt: number, + occurredAt?: number, +): EventEnvelope { + return { + eventId: `${streamId}-${streamVersion}`, + tenantId, + contextId, + streamType, + streamId, + streamVersion, + eventType, + eventSchemaVersion: 1, + operationId: `op-${recordedAt}`, + causedBy: { kind: "command", commandType: "Test" }, + actor: { kind: "human", id: "u-1" }, + recordedAt, + ...(occurredAt === undefined ? {} : { occurredAt }), + payload, + }; +} +const order = ( + version: number, + eventType: string, + payload: unknown, + at: number, +) => + envelope("orders", "order", "o-1", version, eventType, payload, at, at - 1); +const stock = ( + streamId: string, + version: number, + eventType: string, + payload: unknown, + at: number, +) => + envelope("inventory", "stockItem", streamId, version, eventType, payload, at); +// Three streams' histories, each in version order: the order's, and two stock items', one of which +// also allocates to another order and both of which received stock no order names. +const orderStream = [ + order( + 1, + "OrderPlaced", + { + lines: [ + { stockItemId: "s-2", quantity: 1, unitPrice: 5 }, + { stockItemId: "s-1", quantity: 2, unitPrice: 3 }, + { stockItemId: "s-1", quantity: 1, unitPrice: 3 }, + ], + total: 14, + }, + 101, + ), + order(2, "OrderCancelled", {}, 201), +]; +const firstStock = [ + stock("s-1", 1, "StockReceived", { quantity: 10 }, 11), + stock("s-1", 2, "StockAllocated", { orderId: "o-1", quantity: 3 }, 102), + stock("s-1", 3, "AllocationReleased", { orderId: "o-1", quantity: 3 }, 202), +]; +const secondStock = [ + stock("s-2", 1, "StockReceived", { quantity: 5 }, 12), + stock("s-2", 2, "StockAllocated", { orderId: "o-1", quantity: 1 }, 103), + stock("s-2", 3, "StockAllocated", { orderId: "o-2", quantity: 2 }, 150), + stock("s-2", 4, "AllocationReleased", { orderId: "o-1", quantity: 1 }, 203), +]; +// What applyHistoryProjection hands project: the events whose key is the order's. +const ofOrder = (events: readonly EventEnvelope[]) => + events.filter( + (event) => + orderAllocationHistoryKeyOf(tenantId, fromEnvelope(event)) === "o-1", + ); +const streams = [ + ofOrder(orderStream), + ofOrder(firstStock), + ofOrder(secondStock), +]; +// Every interleaving of the streams that keeps each stream's events in version order. +function interleavings( + sequences: readonly (readonly EventEnvelope[])[], +): EventEnvelope[][] { + if (sequences.every((sequence) => sequence.length === 0)) return [[]]; + const result: EventEnvelope[][] = []; + sequences.forEach((sequence, index) => { + const [head, ...rest] = sequence; + if (head === undefined) return; + const remaining = sequences.map((other, at) => + at === index ? rest : other, + ); + for (const tail of interleavings(remaining)) result.push([head, ...tail]); + }); + return result; +} +const expected: OrderAllocationHistoryFields = { + orderId: "o-1", + placedAt: 100, + cancelledAt: 200, + allocations: [ + { stockItemId: "s-1", quantity: 3, allocatedAt: 102, releasedAt: 202 }, + { stockItemId: "s-2", quantity: 1, allocatedAt: 103, releasedAt: 203 }, + ], +}; + +test("pure: the order allocation history keys an order's events by its stream, a stock item's allocation events by their order, and every other event to none", () => { + const [placed] = orderStream; + const [received, allocated] = firstStock; + expect(orderAllocationHistoryKeyOf(tenantId, placed!)).toBe("o-1"); + expect(orderAllocationHistoryKeyOf(tenantId, allocated!)).toBe("o-1"); + expect(orderAllocationHistoryKeyOf(tenantId, secondStock[2]!)).toBe("o-2"); + expect(orderAllocationHistoryKeyOf(tenantId, received!)).toBeNull(); + const baseline = stock( + "s-1", + 4, + "baseline", + { state: {}, stateSchemaVersion: 2 }, + 300, + ); + expect(orderAllocationHistoryKeyOf(tenantId, baseline)).toBeNull(); + expect(streams.map((events) => events.length)).toStrictEqual([2, 2, 2]); +}); + +test("pure: folding one order's events in every interleaving of its three streams, a stock event before OrderPlaced among them, gives one row", () => { + const orders = interleavings(streams); + // 6! / (2! 2! 2!) interleavings, of which those that start with a stock event fold an allocation + // into a row that has no placedAt yet. + expect(orders).toHaveLength(90); + expect( + orders.filter((events) => events[0]?.streamType === "stockItem"), + ).toHaveLength(60); + for (const events of orders) + expect(projectOrderAllocationHistory(tenantId, null, events)).toStrictEqual( + expected, + ); +}); + +test("pure: folding one order's events in one call, one call per event, or two calls split anywhere gives the same row", () => { + for (const events of interleavings(streams)) { + const oneByOne = events.reduce( + (row, event) => projectOrderAllocationHistory(tenantId, row, [event]), + null, + ); + expect(oneByOne).toStrictEqual(expected); + for (let cut = 1; cut < events.length; cut += 1) { + const head = projectOrderAllocationHistory( + tenantId, + null, + events.slice(0, cut), + ); + expect( + projectOrderAllocationHistory(tenantId, head, events.slice(cut)), + ).toStrictEqual(expected); + } + } +}); + +test("pure: a cancelled order keeps its row, and a release after the cancel folds into it", () => { + const [placed, cancelled] = orderStream; + const [, allocated, released] = firstStock; + const cancelledRow = projectOrderAllocationHistory(tenantId, null, [ + placed!, + allocated!, + cancelled!, + ]); + expect(cancelledRow).toStrictEqual({ + orderId: "o-1", + placedAt: 100, + cancelledAt: 200, + allocations: [{ stockItemId: "s-1", quantity: 3, allocatedAt: 102 }], + }); + expect( + projectOrderAllocationHistory(tenantId, cancelledRow, [released!]), + ).toStrictEqual({ + ...cancelledRow, + allocations: [ + { stockItemId: "s-1", quantity: 3, allocatedAt: 102, releasedAt: 202 }, + ], + }); +}); + +test("pure: the fold refuses to start a row with no event of an order", () => { + expect(() => + projectOrderAllocationHistory(tenantId, null, [firstStock[0]!]), + ).toThrow( + "The order allocation history needs a prior row or an event of an order", + ); +}); From 93cadcf93505f4b9759168ead1492d3ef8cc46af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:52:46 +0200 Subject: [PATCH 124/155] design: 4a the broken audit names its table; the tier words Co-Authored-By: Claude Opus 5.5 --- .../baseline-operations.broken-audit-aborts.sdp.md | 6 +++--- .../baseline-operations.broken-metrics-never-abort.sdp.md | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md b/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md index 371c3f1..19e37ff 100644 --- a/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md +++ b/design/specs/operations/baseline-operations.broken-audit-aborts.sdp.md @@ -9,7 +9,7 @@ relations: --- # Break mandatory audit -Sc ALL-1 · native tier · fixture composition · the second of the two enumerated cases; applies to every installed layer. +Sc ALL-1 · native backend tier · fixture composition · the second of the two enumerated cases; applies to every installed layer. The fault sits in the fixture composition's `PlaceOrder`, whose declaration sets `audit` and whose `subjectFrom` gives the order ID `auditFaultOrderId` a subject with one field the `auditRecords` validator does not declare. An ordinary client with grants sends the receipted `PlaceOrder` for that order. The depot creates the order and claims its stock, step 10 inserts the receipt, and Convex refuses the audit record's insert, so the plain error ends the mutation and nothing of the command survives. The fault reads no switch and touches no shared library: authorization compares only the subject's three fields, and every other order ID gets the plain subject. @@ -27,11 +27,11 @@ And the failure is {surfaced: "returned to the caller as a technical failure"} ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The verifier is `tests/native/baseline-operations.broken-audit-aborts.test.ts`, whose `specTest` anchor `test:operations.baseline-operations.broken-audit-aborts` verifies this example. - The first Given step starts `fixtureBackend()`, grants the fixture's `orderPermission` and `permissions.stock` to the human principal `${issuer}|user-1` in the tenants `tenant-all-1` and `sinkFaultTenant` through `grants:grant` with admin access, and, as that caller through an ordinary client carrying the fixture issuer's token for `user-1`, adds 10 units of the product `p-1` in each tenant through `depotCommands:addStock`. - The subsystem step for mandatory audit selects the tenant `tenant-all-1` and the order ID `auditFaultOrderId`. - The When step reads the depot's `events` and `streams` and the parent's `receipts` and `auditRecords` with admin access, takes a log mark, calls `orders:placeOrder` through the ordinary client with the selected tenant, the request key `all-1-key` and the input `{ orderId, title: "Order", lines: [{ productId: "p-1", quantity: 1 }] }`, keeps the answer or the error, and reads the completion records after the mark until one has the identifier `orders:placeOrder`. - The Then step for a command that rolls back asserts that the four tables hold exactly what they held before the call. -- The step for a technical failure asserts that the error is not a `ConvexError`, so `classifyThrown` reads it as technical, that the completion record's `error` contains `auditFault`, that exactly one of its `logLines` contains `diagnostic {` and that line contains `technical`, `tenant-all-1` and `all-1-key` and does not contain `operationId`, and that none of its `logLines` contains `diagnostic gap`. +- The step for a technical failure asserts that the error is not a `ConvexError`, so `classifyThrown` reads it as technical, that the completion record's `error` contains both `auditRecords` and `auditFault`, so the refusal is the audit record's insert and no other write's, that exactly one of its `logLines` contains `diagnostic {` and that line contains `technical`, `tenant-all-1` and `all-1-key` and does not contain `operationId`, and that none of its `logLines` contains `diagnostic gap`. - The test then shows on the same deployment that the audit writer runs: `orders:placeOrder` in `tenant-all-1` with the order ID `order-control` and the request key `all-1-control` returns `applied`, and `auditRecords` holds exactly one record, whose `operationId` is the one returned and whose `decision` is `applied`. diff --git a/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md b/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md index 01240bd..818552d 100644 --- a/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md +++ b/design/specs/operations/baseline-operations.broken-metrics-never-abort.sdp.md @@ -9,7 +9,7 @@ relations: --- # Break metrics and logging -Sc ALL-1 · native tier · fixture composition · the first of the two enumerated cases; applies to every installed layer. +Sc ALL-1 · native backend tier · fixture composition · the first of the two enumerated cases; applies to every installed layer. The fault sits in the fixture composition's diagnostic sink: `PlaceOrder` names `fixtureDiagnosticSink`, which throws for every line of the tenant `sinkFaultTenant`. An ordinary client with grants sends the receipted `PlaceOrder` in that tenant. The command commits its events, its receipt and its audit record; at step 11 the sink throws, `emitDiagnostic` swallows the error and writes one gap line in place of the lost record, and the client receives the applied answer. @@ -27,7 +27,7 @@ And the failure is {surfaced: "reported as a diagnostic gap without touching the ## Verification — executable -- Runs in the native tier on the fixture composition; every test owns its disposable backend. +- Runs in the native backend tier on the fixture composition; every test owns its disposable backend. - The verifier is `tests/native/baseline-operations.broken-metrics-never-abort.test.ts`, whose `specTest` anchor `test:operations.baseline-operations.broken-metrics-never-abort` verifies this example. - The first Given step and the When step are those of `spec:operations.baseline-operations.broken-audit-aborts`. - The subsystem step for metrics and logging selects the tenant `sinkFaultTenant` and the order ID `order-all-1`. From 175e24c75d3619175379ba8c9a700066b377dabd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:53:04 +0200 Subject: [PATCH 125/155] build: hosted hostedTarget, the selection kinds and the planted-key test hostedTarget reads the four variables and refuses by name, never by value, a key that is not a development deploy key and a URL that does not name the deployment the key carries, the regional form the CLI prints accepted. The admin access and the backup archive CLI take a selection kind: the local backend by its arguments, a hosted deployment by CONVEX_DEPLOY_KEY in the child's environment only, with a dotenv file in the CLI's directory refused. createHostedAccess adds the usage and version reads; hostedBackend gives a test the deployment, records each deploy's wall time and refuses to stop, kill or restart it, and fixtureBackend and productionBackend refuse under a hosted run. The pure planted-key test echoes the key's four forms through a stub fetch, socket and CLI on every member and path, and finds none in any error, output, log line or record. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- design/specs/platform/native-harness.sdp.md | 3 +- harness/admin.ts | 126 ++++--- harness/backend.ts | 3 +- harness/backup-archive.ts | 79 +++- harness/hosted.ts | 162 ++++++++ harness/native.ts | 142 ++++++- harness/watch.ts | 7 +- tests/pure/convex-cli.test.ts | 10 +- tests/pure/harness-rules.test.ts | 17 +- tests/pure/hosted.test.ts | 391 +++++++++++++++++++- tests/pure/native.test.ts | 28 +- 12 files changed, 891 insertions(+), 79 deletions(-) create mode 100644 harness/hosted.ts diff --git a/design/README.md b/design/README.md index c70aef1..eb6a08c 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -237 specs · 5 packs · 132 anchors → 374 nodes · 1236 edges (0 errors, 0 warnings) +237 specs · 5 packs · 133 anchors → 375 nodes · 1237 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index e9228dc..3e5a39b 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -95,7 +95,8 @@ The harness is plain test support with no Convex function in it. The entries who - typeHostedUsage: [extension] `interface HostedUsage { readAt: string; seedStatus: string | null; response: JsonValue }` holds the deployment's answer to `GET /api/v1/get_current_usage`, its current UTC day and calendar month, as `convex deployment usage --json` reads it; the endpoint is beta, may understate usage and promises no time by which a call is counted, so a difference of two answers is recorded in the plan's units, never in money, and is the run's own only when no other run on the deployment overlapped it (E-59) - typeHostedTarget: [extension] `interface HostedTarget { url: string; deployment: string; keyName: string; statedPlan: string }`; the deploy key is no member, because the harness's admin access holds it and a test never reads it (E-59) - fnHostedTarget: [extension] `hostedTarget(variables: Readonly>): HostedTarget` reads `HOSTED_DEPLOYMENT_URL`, `HOSTED_DEPLOY_KEY`, `HOSTED_DEPLOY_KEY_NAME` and `HOSTED_PLAN` and refuses as the rules say (E-59) -- fnHostedBackend: [extension] `hostedBackend(options?: { composition?: "fixture" | "production"; issuer?: string; deploy?: boolean }): Promise` deploys the composition unless `deploy` is `false`, sets the fixture issuer's variables, records the deploy's wall time and, when the test finishes, closes its clients; `stop`, `kill` and `restart` refuse, `dispose` removes nothing on the deployment, and reading `adminKey` throws (E-59, E-13) +- typeHostedAccess: [extension] `interface HostedAccess extends AdminAccess { usage(): Promise; instanceVersion(): Promise }` is the admin access to a hosted deployment, which holds the deploy key and a fresh home; `usage` reads `GET /api/v1/get_current_usage` with the key, and `instanceVersion` is the answer of `/instance_version` or `null` (E-59) +- fnHostedBackend: [extension] `hostedBackend(options?: { composition?: "fixture" | "production"; issuer?: string; deploy?: boolean }): Promise` sets the fixture issuer's variables, deploys the composition unless `deploy` is `false`, records the wall time of each deploy on the running test, which the record gathers into `deploys`, and, when the test finishes, closes its clients; `stop`, `kill` and `restart` refuse, `dispose` removes nothing on the deployment, and reading `adminKey` throws (E-59, E-13) - hostedSelection: [extension] a CLI call of a native run on a hosted deployment runs in the composition's project directory with the environment `PATH`, `HOME` set to a fresh directory, `TMPDIR` and `CONVEX_DEPLOY_KEY`, and no `--url` or `--admin-key`; `convex` 1.46.0 resolves a development deployment's deploy key to its URL and uses the key itself as the deployment's credential, in `node_modules/convex/src/cli/lib/deploymentSelection.ts` lines 714 to 749, and sends it as `Authorization: Convex` in `node_modules/convex/src/cli/lib/utils/utils.ts` lines 1373 to 1389; the local path keeps its arguments (E-59) - deployKeyActions: [extension] the deploy key allows exactly `deployment:deploy`, `deployment:env:view`, `deployment:env:write`, `deployment:data:view`, `deployment:data:write`, `deployment:functions:runInternalQueries`, `deployment:functions:runInternalMutations`, `deployment:functions:runInternalActions`, `deployment:logs:view`, `deployment:usage:view`, `deployment:backups:view`, `deployment:backups:create`, `deployment:backups:download` and `deployment:backups:import`, the actions the routes of `AdminAccess` and the CLI's export and import check in the upstream backend at `5c7cb5b`, and not `deployment:functions:actAsUser`, pause, usage limits, integrations or the audit log (E-59) - hostedRedaction: [extension] the secrets of a native run on a hosted deployment are the four forms of the deploy key, given to every child, every buffer and the reporter; the admin access built over a hosted deployment catches every rejection of its members, the HTTP client's raw response text and its `Invalid response` text among them, a fetch that rejects and a call that aborts, and rethrows it with those forms replaced; its HTTP and socket clients take a logger that replaces them; a CLI child's output is redacted on success as on failure and on an abort; the reporter redacts the serialized record before it writes it; the import upload token, the export download token, application data and the fixture issuer's private key are not among these secrets and are named as not covered, not scrubbed as if they were (E-59, E-15) diff --git a/harness/admin.ts b/harness/admin.ts index e02b600..e126fac 100644 --- a/harness/admin.ts +++ b/harness/admin.ts @@ -7,10 +7,15 @@ import { BaseConvexClient, ConvexHttpClient } from "convex/browser"; import { makeFunctionReference } from "convex/server"; import type { Value } from "convex/values"; import { tmpdir } from "node:os"; -import { join } from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { redact, redactError, redactingLogger, runChild } from "./child.js"; -import { isolatedCli, createBackupArchiveAccess } from "./backup-archive.js"; +import { + cliSelection, + convexCliScript, + createBackupArchiveAccess, + isolatedCli, + refuseDotenv, +} from "./backup-archive.js"; import type { CliOutput } from "./backup-archive.js"; import { projectDirectory } from "./composition.js"; import type { Composition } from "./composition.js"; @@ -20,9 +25,14 @@ const anchor = codeAnchor({ satisfies: ref("spec:platform.native-harness"), }); void anchor; +// How the harness selects the backend it acts on: the local backend it started, by its URL and admin +// key, or a hosted deployment, by its URL and deploy key. A CLI call on a hosted deployment gets the +// key only as CONVEX_DEPLOY_KEY in its environment, never as an argument. +export type Selection = + | { kind: "local"; url: string; adminKey: string } + | { kind: "hosted"; url: string; key: string }; export interface AdminTarget { - url: string; - adminKey: string; + selection: Selection; home: string; // The composition that deploy and codegen act on. composition: Composition; @@ -96,7 +106,20 @@ export interface CliCall { // Infinity for a call that runs until it is stopped. runChild refuses it. timeoutMs: number; } -const repositoryRoot = join(import.meta.dirname, ".."); +// What a test of the admin access replaces: the fetch of its HTTP calls, the socket client of its +// system reads and the CLI script its children run. +export type SocketFactory = ( + url: string, + onTransition: (tokens: string[]) => void, + logger: ReturnType, +) => AdminSocketClient; +export interface AdminDependencies { + fetch?: typeof globalThis.fetch; + socket?: SocketFactory; + cliScript?: string; +} +const credential = (selection: Selection) => + selection.kind === "local" ? selection.adminKey : selection.key; const commandArgs = { deploy: ["deploy", "--yes", "--codegen", "disable", "--typecheck", "disable"], codegen: ["codegen", "--typecheck", "disable"], @@ -109,19 +132,23 @@ const commandTimeoutMs = { } as const; // The CLI runs in the composition's project directory, the one place it reads convex.json from. export function convexCli( - target: Pick, + target: Pick, command: "deploy" | "codegen" | "dev", + script = convexCliScript, ): CliCall { - const selection = ["--url", target.url, "--admin-key", target.adminKey]; + const selection = cliSelection(target.selection); + const cwd = projectDirectory(target.composition); + if (target.selection.kind === "hosted") refuseDotenv(cwd); return { file: process.execPath, - args: [ - join(repositoryRoot, "node_modules/convex/bin/main.js"), - ...commandArgs[command], - ...selection, - ], - cwd: projectDirectory(target.composition), - env: { PATH: process.env.PATH ?? "", HOME: target.home, TMPDIR: tmpdir() }, + args: [script, ...commandArgs[command], ...selection.args], + cwd, + env: { + PATH: process.env.PATH ?? "", + HOME: target.home, + TMPDIR: tmpdir(), + ...selection.env, + }, timeoutMs: commandTimeoutMs[command], }; } @@ -134,7 +161,7 @@ type AdminHttpClient = ConvexHttpClient & { args: Record, ): Promise; }; -type AdminSocketClient = { +export type AdminSocketClient = { setAdminAuth(key: string): void; subscribe( name: string, @@ -147,16 +174,29 @@ type AdminSocketClient = { export function createAdminAccess( target: AdminTarget, state: AdminState, + dependencies: AdminDependencies = {}, ): AdminAccess { const marks = new WeakMap(); const logger = redactingLogger(target.secrets); - const http = new ConvexHttpClient(target.url, { - logger, - }) as AdminHttpClient; - http.setAdminAuth(target.adminKey); - const authorization = { Authorization: `Convex ${target.adminKey}` }; + const url = target.selection.url; + const hosted = target.selection.kind === "hosted"; + // Read at each call, so that a test that replaces the global fetch reaches every call. + const fetch: typeof globalThis.fetch = (input, init) => + (dependencies.fetch ?? globalThis.fetch)(input, init); + const http = new ConvexHttpClient(url, { logger, fetch }) as AdminHttpClient; + http.setAdminAuth(credential(target.selection)); + const authorization = { + Authorization: `Convex ${credential(target.selection)}`, + }; + const socket: SocketFactory = + dependencies.socket ?? + ((socketUrl, onTransition, socketLogger) => + new BaseConvexClient(socketUrl, onTransition, { + unsavedChangesWarning: false, + logger: socketLogger, + }) as unknown as AdminSocketClient); async function cli(command: "deploy" | "codegen") { - const call = convexCli(target, command); + const call = convexCli(target, command, dependencies.cliScript); // CLI children are short and bounded; they lose their backend when a run is swept. await runChild(`convex ${command}`, call.file, call.args, { cwd: call.cwd, @@ -181,8 +221,8 @@ export function createAdminAccess( void client.close(); finish(); }; - const client = new BaseConvexClient( - target.url, + const client = socket( + url, (tokens) => { for (const token of tokens) { let value: Value | undefined; @@ -194,8 +234,8 @@ export function createAdminAccess( if (value !== undefined) return settle(() => resolve(value)); } }, - { unsavedChangesWarning: false, logger }, - ) as unknown as AdminSocketClient; + logger, + ); const timer = setTimeout( () => settle(() => @@ -205,7 +245,7 @@ export function createAdminAccess( ), timeoutMs, ); - client.setAdminAuth(target.adminKey); + client.setAdminAuth(credential(target.selection)); client.subscribe( name, args, @@ -214,7 +254,7 @@ export function createAdminAccess( }); } return guarded(target.secrets, { - ...createBackupArchiveAccess(target), + ...createBackupArchiveAccess(target, dependencies.cliScript), async deploy() { await cli("deploy"); state.deployed = target.composition; @@ -228,26 +268,24 @@ export function createAdminAccess( [...commandArgs.deploy], signals.length === 0 ? undefined : AbortSignal.any(signals), directory, + dependencies.cliScript, ); state.deployed = undefined; return result; }, codegen: () => cli("codegen"), async setEnvironment(variables) { - const response = await fetch( - `${target.url}/api/update_environment_variables`, - { - method: "POST", - headers: { ...authorization, "Content-Type": "application/json" }, - body: JSON.stringify({ - changes: Object.entries(variables).map(([name, value]) => ({ - name, - value, - })), - }), - signal: AbortSignal.timeout(10000), - }, - ); + const response = await fetch(`${url}/api/update_environment_variables`, { + method: "POST", + headers: { ...authorization, "Content-Type": "application/json" }, + body: JSON.stringify({ + changes: Object.entries(variables).map(([name, value]) => ({ + name, + value, + })), + }), + signal: AbortSignal.timeout(10000), + }); if (!response.ok) throw new Error( `Setting environment variables failed with status ${response.status}: ${redact(await response.text(), target.secrets)}`, @@ -369,7 +407,7 @@ export function createAdminAccess( let response: Response; try { response = await fetch( - `${target.url}/api/stream_function_logs?cursor=${cursor}`, + `${url}/api/stream_function_logs?cursor=${cursor}`, { headers: authorization, signal: AbortSignal.timeout(left) }, ); } catch (error) { @@ -385,7 +423,9 @@ export function createAdminAccess( entries: { kind: string }[]; newCursor: number; }; - if (body.entries.length >= 1000) + // The local backend keeps its last 1000 entries. A hosted deployment's bound is not + // published, so the hosted path reads as it goes and promotes no local bound. + if (!hosted && body.entries.length >= 1000) throw new Error( `One function log read returned ${body.entries.length} entries. The backend keeps its last 1000, so records may be lost. Read the log more often.`, ); diff --git a/harness/backend.ts b/harness/backend.ts index 35c7c27..fa6d162 100644 --- a/harness/backend.ts +++ b/harness/backend.ts @@ -238,8 +238,7 @@ export async function startBackend(options: StartOptions): Promise { const url = `http://127.0.0.1:${port}`; const access = createAdminAccess( { - url, - adminKey, + selection: { kind: "local", url, adminKey }, home, composition: options.composition ?? fixtureComposition, secrets, diff --git a/harness/backup-archive.ts b/harness/backup-archive.ts index 9046904..9ade299 100644 --- a/harness/backup-archive.ts +++ b/harness/backup-archive.ts @@ -3,11 +3,13 @@ import { codeAnchorId, ref, } from "@libar-dev/software-delivery-protocol"; +import { existsSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import type { AdminAccess, AdminTarget } from "./admin.js"; +import type { AdminAccess, AdminTarget, Selection } from "./admin.js"; import type { Backend } from "./backend.js"; +import { projectDirectory } from "./composition.js"; import { redact, runChild } from "./child.js"; const anchor = codeAnchor({ id: codeAnchorId("impl:platform.native-harness.backup-archive"), @@ -16,34 +18,67 @@ const anchor = codeAnchor({ }); void anchor; +export const convexCliScript = join( + import.meta.dirname, + "../node_modules/convex/bin/main.js", +); +// The Convex CLI loads a .env or a .env.local from its working directory, and either can select +// another deployment, so a CLI call on a hosted deployment refuses a directory that holds one. The +// refusal names the file and never what it holds. +export function refuseDotenv(directory: string): void { + for (const name of [".env", ".env.local"]) { + const file = join(directory, name); + if (existsSync(file)) + throw new Error( + `${file} exists, and the Convex CLI would load it. A native run on a hosted deployment refuses a composition directory that holds .env or .env.local.`, + ); + } +} +// The CLI's selection of the backend: arguments for the local backend, one environment variable +// for a hosted deployment. +export function cliSelection(selection: Selection): { + args: string[]; + env: Record; +} { + return selection.kind === "local" + ? { + args: ["--url", selection.url, "--admin-key", selection.adminKey], + env: {}, + } + : { args: [], env: { CONVEX_DEPLOY_KEY: selection.key } }; +} export interface CliOutput { command: string[]; stdout: string; stderr: string; } -// Uses the installed CLI and an isolated home, with no inherited deployment selection. +// Uses the installed CLI and an isolated home, with no inherited deployment selection: the local +// backend by its arguments, a hosted deployment by CONVEX_DEPLOY_KEY in the child's environment. export async function isolatedCli( - backend: Pick, + target: Pick, args: string[], signal: AbortSignal | undefined, directory = join(import.meta.dirname, ".."), + script = convexCliScript, ): Promise { + if (target.selection.kind === "hosted") refuseDotenv(directory); const home = await mkdtemp(join(tmpdir(), "convex-cli-")); - const selection = ["--url", backend.url, "--admin-key", backend.adminKey]; + const selection = cliSelection(target.selection); const file = process.execPath; - const command = [ - join(import.meta.dirname, "../node_modules/convex/bin/main.js"), - ...args, - ...selection, - ]; - const safe = (value: string) => redact(value, [backend.adminKey]); + const command = [script, ...args, ...selection.args]; + const safe = (value: string) => redact(value, target.secrets); try { const output = await runChild(`convex ${args[0]}`, file, command, { cwd: directory, - env: { PATH: process.env.PATH ?? "", HOME: home, TMPDIR: tmpdir() }, + env: { + PATH: process.env.PATH ?? "", + HOME: home, + TMPDIR: tmpdir(), + ...selection.env, + }, timeoutMs: 60000, - secrets: [backend.adminKey], + secrets: target.secrets, ...(signal === undefined ? {} : { signal }), output: "both", }); @@ -62,18 +97,34 @@ export function deployTemporaryCopy( return backend.admin.deployTemporary(directory, signal); } +// On the local backend the export and the import run at the repository root; on a hosted +// deployment they run in the composition's project directory, the one its refusal of a dotenv file +// covers. export function createBackupArchiveAccess( - target: Pick, + target: Pick, + script?: string, ): Pick { + const repository = + target.selection.kind === "hosted" + ? projectDirectory(target.composition) + : join(import.meta.dirname, ".."); return { async exportBackupArchive(path) { - await isolatedCli(target, ["export", "--path", path], target.signal); + await isolatedCli( + target, + ["export", "--path", path], + target.signal, + repository, + script, + ); }, async importBackupArchive(path) { await isolatedCli( target, ["import", "--replace", "--yes", path], target.signal, + repository, + script, ); }, }; diff --git a/harness/hosted.ts b/harness/hosted.ts new file mode 100644 index 0000000..26e6123 --- /dev/null +++ b/harness/hosted.ts @@ -0,0 +1,162 @@ +import { + codeAnchor, + codeAnchorId, + ref, +} from "@libar-dev/software-delivery-protocol"; +import { createAdminAccess, guarded } from "./admin.js"; +import type { AdminAccess, AdminDependencies, AdminState } from "./admin.js"; +import type { Composition } from "./composition.js"; +import type { HostedUsage, JsonValue } from "./evidence.js"; +const anchor = codeAnchor({ + id: codeAnchorId("impl:platform.native-harness.hosted"), + label: "the selection of a hosted deployment and its admin access", + satisfies: ref("spec:platform.native-harness"), +}); +void anchor; +// What a native run on a hosted deployment knows of it. The deploy key is no member: the admin +// access holds it, and a test never reads it. +export interface HostedTarget { + url: string; + deployment: string; + keyName: string; + statedPlan: string; +} +// What the hosted run's global setup gives every test: the target and when the run started. +export interface HostedRun { + target: HostedTarget; + startedAt: string; +} +declare module "vitest" { + interface ProvidedContext { + hostedRun: HostedRun; + } +} +type Variables = Readonly>; +// The four process variables a hosted run reads. Only the deploy key is a secret. +export const hostedVariables = [ + "HOSTED_DEPLOYMENT_URL", + "HOSTED_DEPLOY_KEY", + "HOSTED_DEPLOY_KEY_NAME", + "HOSTED_PLAN", +] as const; +// Every refusal names the variable and never its value. +function read(variables: Variables, name: (typeof hostedVariables)[number]) { + const value = variables[name]; + if (value === undefined || value.trim() === "") + throw new Error( + `${name} is not set. A native run on a hosted deployment reads ${hostedVariables.join(", ")} from the process environment.`, + ); + return value; +} +const deploymentName = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; +// The deployment a development deploy key carries: dev:, the deployment's name, | and the secret. +function keyDeployment(key: string): string { + if (!key.startsWith("dev:")) + throw new Error( + "HOSTED_DEPLOY_KEY does not begin dev:. A native run on a hosted deployment takes a development deploy key and refuses a production, preview or project key.", + ); + const bar = key.indexOf("|"); + const name = key.slice("dev:".length, bar); + if (bar === -1 || !deploymentName.test(name) || bar === key.length - 1) + throw new Error( + "HOSTED_DEPLOY_KEY is not dev:, a deployment name, | and a secret.", + ); + return name; +} +// Reads the four variables and refuses a key that is not a development deploy key, and a URL +// that does not name the deployment the key carries: https://, the name, optionally the region +// the CLI prints, and .convex.cloud. +export function hostedTarget(variables: Variables): HostedTarget { + const url = read(variables, "HOSTED_DEPLOYMENT_URL"); + const key = read(variables, "HOSTED_DEPLOY_KEY"); + const keyName = read(variables, "HOSTED_DEPLOY_KEY_NAME"); + const statedPlan = read(variables, "HOSTED_PLAN"); + const deployment = keyDeployment(key); + const printed = new RegExp( + `^https://${deployment}(?:\\.[a-z0-9]+(?:-[a-z0-9]+)*)?\\.convex\\.cloud$`, + ); + if (!printed.test(url)) + throw new Error( + "HOSTED_DEPLOYMENT_URL does not name the deployment HOSTED_DEPLOY_KEY carries: it is not https://, that deployment's name, an optional region and .convex.cloud.", + ); + return { url, deployment, keyName, statedPlan }; +} +// The secrets of a hosted run: the whole key, its part after the |, and both URL-encoded. +export function deployKeyForms(key: string): string[] { + const secret = key.slice(key.indexOf("|") + 1); + return [ + ...new Set( + [key, secret, encodeURIComponent(key), encodeURIComponent(secret)].filter( + (form) => form !== "", + ), + ), + ]; +} +// The admin access to a hosted deployment, and the two reads only a hosted deployment answers. +export interface HostedAccess extends AdminAccess { + // The deployment's answer to GET /api/v1/get_current_usage, read with the deploy key. + usage(): Promise; + // The answer of /instance_version, or null when the deployment gives none. + instanceVersion(): Promise; +} +export function createHostedAccess(options: { + variables: Variables; + composition: Composition; + home: string; + state: AdminState; + signal?: AbortSignal; + dependencies?: AdminDependencies; +}): { target: HostedTarget; access: HostedAccess; secrets: string[] } { + const target = hostedTarget(options.variables); + const key = read(options.variables, "HOSTED_DEPLOY_KEY"); + const secrets = deployKeyForms(key); + const dependencies = options.dependencies ?? {}; + const fetch: typeof globalThis.fetch = (input, init) => + (dependencies.fetch ?? globalThis.fetch)(input, init); + const access = createAdminAccess( + { + selection: { kind: "hosted", url: target.url, key }, + home: options.home, + composition: options.composition, + secrets, + ...(options.signal === undefined ? {} : { signal: options.signal }), + }, + options.state, + dependencies, + ); + const hosted: HostedAccess = guarded(secrets, { + ...access, + async usage() { + const response = await fetch(`${target.url}/api/v1/get_current_usage`, { + headers: { Authorization: `Convex ${key}` }, + signal: AbortSignal.timeout(30000), + }); + const readAt = new Date().toISOString(); + const text = await response.text(); + if (!response.ok) + throw new Error( + `Reading the deployment's usage failed with status ${response.status}: ${text}`, + ); + const answer = JSON.parse(text) as JsonValue; + const seedStatus = + typeof answer === "object" && + answer !== null && + !Array.isArray(answer) && + typeof answer["seedStatus"] === "string" + ? answer["seedStatus"] + : null; + return { readAt, seedStatus, response: answer }; + }, + async instanceVersion() { + try { + const response = await fetch(`${target.url}/instance_version`, { + signal: AbortSignal.timeout(10000), + }); + return response.ok ? (await response.text()).trim() : null; + } catch { + return null; + } + }, + }); + return { target, access: hosted, secrets }; +} diff --git a/harness/native.ts b/harness/native.ts index 9306d51..9189f99 100644 --- a/harness/native.ts +++ b/harness/native.ts @@ -3,13 +3,20 @@ import { codeAnchorId, ref, } from "@libar-dev/software-delivery-protocol"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { TestRunner, inject, onTestFinished } from "vitest"; +import type { AdminState } from "./admin.js"; import { startBackend } from "./backend.js"; import { fixtureComposition, productionComposition } from "./composition.js"; import type { Composition } from "./composition.js"; import type { Executable } from "./executable.js"; import type { Backend } from "./backend.js"; -import type { JsonValue, NativeTestFacts } from "./evidence.js"; +import { hostedDeployMeasurement } from "./evidence.js"; +import type { HostedDeploy, JsonValue, NativeTestFacts } from "./evidence.js"; +import { createHostedAccess } from "./hosted.js"; +import type { HostedAccess, HostedRun } from "./hosted.js"; import { createFixtureIssuer } from "./identity.js"; // Binds this file to its Spec in the corpus graph. Every other file under harness/ carries its own anchor. const anchor = codeAnchor({ @@ -22,7 +29,7 @@ function current() { const test = TestRunner.getCurrentTest(); if (test === undefined) throw new Error( - "fixtureBackend(), productionBackend() and measure() must run inside a test", + "fixtureBackend(), productionBackend(), hostedBackend() and measure() must run inside a test", ); const facts: NativeTestFacts = (test.meta.native ??= { backends: [], @@ -57,6 +64,10 @@ async function compositionBackend( }, ): Promise { const { test, facts } = current(); + if ((inject("hostedRun") as HostedRun | undefined) !== undefined) + throw new Error( + "fixtureBackend() and productionBackend() start a local backend, which a native run on a hosted deployment refuses. Use hostedBackend().", + ); const run = inject("nativeRun"); const issuer = await createFixtureIssuer(options.issuer); let owned: Backend | undefined; @@ -86,6 +97,133 @@ async function compositionBackend( }); return starting; } +// The hosted deployment as a backend of one test: its admin access holds the deploy key, it starts +// and removes nothing, and its facts say what the test deployed and that the data is kept. +export interface HostedBackend extends Backend { + readonly admin: HostedAccess; +} +// Sets the fixture issuer's variables on the hosted deployment, deploys the composition unless +// deploy is false, records each deploy's wall time on the running test and, when the test +// finishes, closes its clients. stop, kill and restart refuse, dispose removes nothing on the +// deployment, and reading adminKey throws. +export async function hostedBackend( + options: { + composition?: "fixture" | "production"; + issuer?: string; + deploy?: boolean; + } = {}, +): Promise { + const { test, facts } = current(); + if ((inject("hostedRun") as HostedRun | undefined) === undefined) + throw new Error( + "hostedBackend() runs only under npm run test:hosted, whose global setup selects the hosted deployment", + ); + const composition = + options.composition === "production" + ? productionComposition + : fixtureComposition; + const issuer = await createFixtureIssuer(options.issuer); + const home = await mkdtemp(join(tmpdir(), "libar-hosted-")); + const state: AdminState = { + deployed: undefined, + environment: new Set(), + logProcess: {}, + }; + let identityIssuer = issuer.issuer; + let disposed = false; + let access: HostedAccess; + let url: string; + try { + const created = createHostedAccess({ + variables: process.env, + composition, + home, + state, + signal: test.context.signal, + }); + access = created.access; + url = created.target.url; + } catch (error) { + await rm(home, { recursive: true, force: true }); + throw error; + } + const deployed = (deploy: HostedDeploy) => + current().facts.measurements.push({ + name: hostedDeployMeasurement, + value: { ...deploy }, + }); + const assertLive = () => { + if (disposed) throw new Error("The hosted backend is disposed"); + }; + const admin: HostedAccess = { + ...access, + async deploy() { + assertLive(); + const start = performance.now(); + await access.deploy(); + deployed({ + composition: composition.name, + wallMs: performance.now() - start, + }); + }, + async deployTemporary(directory, signal) { + assertLive(); + const start = performance.now(); + const result = await access.deployTemporary(directory, signal); + deployed({ composition: null, wallMs: performance.now() - start }); + return result; + }, + async setEnvironment(variables) { + assertLive(); + await access.setEnvironment(variables); + identityIssuer = variables.AUTH_ISSUER ?? identityIssuer; + }, + }; + const refuse = (what: string) => async () => { + throw new Error( + `The harness does not ${what} a hosted deployment; it started no process there`, + ); + }; + const backend: HostedBackend = { + url, + get adminKey(): string { + throw new Error( + "A hosted deployment has no admin key in this harness; its deploy key stays inside the admin access", + ); + }, + issuer, + admin, + stop: refuse("stop"), + kill: refuse("kill"), + restart: refuse("restart"), + async dispose() { + if (disposed) return; + disposed = true; + state.logProcess = undefined; + await rm(home, { recursive: true, force: true }); + }, + facts: () => ({ + composition: state.deployed?.name ?? null, + installedLayers: [...(state.deployed?.installedLayers ?? [])], + target: "hosted deployment", + executable: null, + identitySource: { kind: "fixture issuer", issuer: identityIssuer }, + environment: [...state.environment].sort(), + dataset: "kept from earlier runs", + }), + }; + onTestFinished(async () => { + facts.backends.push(backend.facts()); + await backend.dispose(); + }); + await admin.setEnvironment({ + AUTH_ISSUER: issuer.issuer, + AUTH_APPLICATION_ID: issuer.applicationID, + AUTH_JWKS: issuer.jwks, + }); + if (options.deploy !== false) await admin.deploy(); + return backend; +} export function measure(name: string, value: JsonValue): void { current().facts.measurements.push({ name, value }); } diff --git a/harness/watch.ts b/harness/watch.ts index a6e844a..54902e2 100644 --- a/harness/watch.ts +++ b/harness/watch.ts @@ -193,8 +193,11 @@ export async function watch(options: WatchOptions): Promise { child = steps.spawn( convexCli( { - url: backend.url, - adminKey: backend.adminKey, + selection: { + kind: "local", + url: backend.url, + adminKey: backend.adminKey, + }, home, composition: options.composition, }, diff --git a/tests/pure/convex-cli.test.ts b/tests/pure/convex-cli.test.ts index 6244253..83ae08c 100644 --- a/tests/pure/convex-cli.test.ts +++ b/tests/pure/convex-cli.test.ts @@ -5,9 +5,13 @@ import { afterEach, expect, test, vi } from "vitest"; import { convexCli } from "../../harness/admin.js"; import { compositions, fixtureComposition } from "../../harness/composition.js"; import { runChild } from "../../harness/child.js"; -const target = { +const selection = { + kind: "local" as const, url: "http://127.0.0.1:1", adminKey: "the-admin-key", +}; +const target = { + selection, home: "/the/backend/home", composition: fixtureComposition, }; @@ -22,9 +26,9 @@ test.each(["deploy", "codegen", "dev"] as const)( expect(call.args[1]).toBe(command); expect(call.args.slice(-4)).toEqual([ "--url", - target.url, + selection.url, "--admin-key", - target.adminKey, + selection.adminKey, ]); expect(Object.keys(call.env).sort()).toEqual(["HOME", "PATH", "TMPDIR"]); expect(call.env.HOME).toBe(target.home); diff --git a/tests/pure/harness-rules.test.ts b/tests/pure/harness-rules.test.ts index cd5284c..e5f268e 100644 --- a/tests/pure/harness-rules.test.ts +++ b/tests/pure/harness-rules.test.ts @@ -152,8 +152,11 @@ test("pure: function log marks reject a changed process even when the predicate }; const admin = createAdminAccess( { - url: "http://127.0.0.1:1", - adminKey: "dummy", + selection: { + kind: "local", + url: "http://127.0.0.1:1", + adminKey: "dummy", + }, home: "unused", composition: fixtureComposition, secrets: [], @@ -205,8 +208,11 @@ test("pure: every TypeScript file under harness/ reaches a Spec through its own function testAdmin(signal?: AbortSignal) { return createAdminAccess( { - url: "http://127.0.0.1:1", - adminKey: "dummy", + selection: { + kind: "local", + url: "http://127.0.0.1:1", + adminKey: "dummy", + }, home: "unused", composition: fixtureComposition, secrets: ["dummy"], @@ -324,8 +330,7 @@ test("pure: every admin member's rejection and every client log line has its sec const key = "REVIEW_DUMMY_ADMIN_SECRET_0123456789"; const admin = createAdminAccess( { - url: "http://127.0.0.1:1", - adminKey: key, + selection: { kind: "local", url: "http://127.0.0.1:1", adminKey: key }, home: "unused", composition: fixtureComposition, secrets: [key], diff --git a/tests/pure/hosted.test.ts b/tests/pure/hosted.test.ts index 4353c7e..05f2bfb 100644 --- a/tests/pure/hosted.test.ts +++ b/tests/pure/hosted.test.ts @@ -3,11 +3,25 @@ import { specTest, testAnchorId, } from "@libar-dev/software-delivery-protocol"; -import { mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { expect, onTestFinished, test } from "vitest"; +import { join, relative } from "node:path"; +import { format } from "node:util"; +import { expect, onTestFinished, test, vi } from "vitest"; import { answerAcceptance } from "../../harness/acceptance.js"; +import { convexCli } from "../../harness/admin.js"; +import type { AdminSocketClient, SocketFactory } from "../../harness/admin.js"; +import { refuseDotenv } from "../../harness/backup-archive.js"; +import * as child from "../../harness/child.js"; +import type { Composition } from "../../harness/composition.js"; import type { ScenarioGraph } from "../../harness/acceptance.js"; import EvidenceReporter, { amendRunRecord, @@ -15,6 +29,11 @@ import EvidenceReporter, { openRunRecord, } from "../../harness/evidence.js"; import type { NativeRunRecord } from "../../harness/evidence.js"; +import { + createHostedAccess, + deployKeyForms, + hostedTarget, +} from "../../harness/hosted.js"; // Binds this file to the harness Spec, whose rules for a native run on a hosted deployment it // checks with no network and no deploy key. const anchor = specTest({ @@ -173,3 +192,369 @@ test("pure: the acceptance check reads no record of a native run on a hosted dep `acceptance: cannot answer · The record ${join(directory, "hosted.json")} is not a run on the local backend, and a native run on a hosted deployment passes no scenario`, ]); }); + +// A key in the form of a development deploy key, whose secret changes when it is URL-encoded, so +// that its four forms differ. +const plantedKey = () => + `dev:happy-animal-123|pl+an/ted=${randomUUID().replaceAll("-", "")}`; +const variables = (key: string) => ({ + HOSTED_DEPLOYMENT_URL: "https://happy-animal-123.eu-west-1.convex.cloud", + HOSTED_DEPLOY_KEY: key, + HOSTED_DEPLOY_KEY_NAME: "hosted-ci-scoped", + HOSTED_PLAN: "Professional", +}); + +test("pure: hostedTarget refuses each wrong variable by its name and never prints a value", () => { + const key = plantedKey(); + const good = variables(key); + expect(hostedTarget(good)).toEqual({ + url: "https://happy-animal-123.eu-west-1.convex.cloud", + deployment: "happy-animal-123", + keyName: "hosted-ci-scoped", + statedPlan: "Professional", + }); + expect( + hostedTarget({ + ...good, + HOSTED_DEPLOYMENT_URL: "https://happy-animal-123.convex.cloud", + }).url, + ).toBe("https://happy-animal-123.convex.cloud"); + const cases: [Record, string][] = [ + [{ HOSTED_DEPLOYMENT_URL: undefined }, "HOSTED_DEPLOYMENT_URL is not set"], + [{ HOSTED_DEPLOY_KEY: "" }, "HOSTED_DEPLOY_KEY is not set"], + [{ HOSTED_DEPLOY_KEY_NAME: " " }, "HOSTED_DEPLOY_KEY_NAME is not set"], + [{ HOSTED_PLAN: undefined }, "HOSTED_PLAN is not set"], + [ + { HOSTED_DEPLOY_KEY: key.replace("dev:", "prod:") }, + "HOSTED_DEPLOY_KEY does not begin dev:", + ], + [ + { HOSTED_DEPLOY_KEY: key.replace("dev:", "preview:team:") }, + "HOSTED_DEPLOY_KEY does not begin dev:", + ], + [ + { HOSTED_DEPLOY_KEY: "dev:happy-animal-123" }, + "HOSTED_DEPLOY_KEY is not dev:, a deployment name, | and a secret", + ], + [ + { HOSTED_DEPLOY_KEY: "dev:happy-animal-123|" }, + "HOSTED_DEPLOY_KEY is not dev:, a deployment name, | and a secret", + ], + ...[ + "http://happy-animal-123.convex.cloud", + "https://other-animal-456.convex.cloud", + "https://happy-animal-123.eu-west-1.convex.cloud/", + "https://happy-animal-123.convex.site", + "https://happy-animal-1234.convex.cloud", + "https://xhappy-animal-123.convex.cloud", + "https://happy-animal-123.a.b.convex.cloud", + ].map((url): [Record, string] => [ + { HOSTED_DEPLOYMENT_URL: url }, + "HOSTED_DEPLOYMENT_URL does not name the deployment HOSTED_DEPLOY_KEY carries", + ]), + ]; + for (const [change, refusal] of cases) { + const given = { ...good, ...change }; + let message = ""; + try { + hostedTarget(given); + } catch (error) { + message = (error as Error).message; + } + expect(message).toContain(refusal); + for (const value of Object.values(given)) + if (value !== undefined && value.trim() !== "") + expect(message).not.toContain(value); + for (const form of deployKeyForms(key)) expect(message).not.toContain(form); + } +}); + +test("pure: the secrets of a hosted run are the key, its part after the |, and both URL-encoded", () => { + const key = plantedKey(); + const secret = key.slice(key.indexOf("|") + 1); + expect(deployKeyForms(key)).toEqual([ + key, + secret, + encodeURIComponent(key), + encodeURIComponent(secret), + ]); +}); + +test("pure: a CLI call on a hosted deployment refuses a directory with .env or .env.local, naming the file and not its content", async () => { + const directory = await temporary("libar-hosted-dotenv-"); + expect(() => refuseDotenv(directory)).not.toThrow(); + for (const name of [".env", ".env.local"]) { + await writeFile(join(directory, name), "CONVEX_DEPLOYMENT=dev:elsewhere\n"); + let message = ""; + try { + refuseDotenv(directory); + } catch (error) { + message = (error as Error).message; + } + expect(message).toContain(join(directory, name)); + expect(message).not.toContain("elsewhere"); + await rm(join(directory, name)); + } +}); + +// A composition whose project directory is a temporary one, so no file of the repository's root +// takes part. +async function temporaryComposition(): Promise { + const directory = await temporary("libar-hosted-composition-"); + return { + name: "fixture", + project: relative(join(import.meta.dirname, "../.."), directory), + functions: "convex", + installedLayers: [], + }; +} + +test("pure: no CLI call of the hosted path names the backend by an argument, and each gets the key only in its environment", async () => { + const key = plantedKey(); + const composition = await temporaryComposition(); + for (const command of ["deploy", "codegen", "dev"] as const) { + const call = convexCli( + { + selection: { kind: "hosted", url: "unused", key }, + home: "/h", + composition, + }, + command, + ); + expect(call.args).not.toContain("--admin-key"); + expect(call.args).not.toContain("--url"); + expect(Object.keys(call.env).sort()).toEqual([ + "CONVEX_DEPLOY_KEY", + "HOME", + "PATH", + "TMPDIR", + ]); + expect(call.env.CONVEX_DEPLOY_KEY).toBe(key); + } + const runner = vi + .spyOn(child, "runChild") + .mockResolvedValue({ stdout: "", stderr: "" }); + try { + const { access } = createHostedAccess({ + variables: variables(key), + composition, + home: await temporary("libar-hosted-home-"), + state: { deployed: undefined, environment: new Set() }, + }); + await access.deploy(); + await access.codegen(); + await access.deployTemporary(await temporary("libar-hosted-copy-")); + await access.exportBackupArchive("archive.zip"); + await access.importBackupArchive("archive.zip"); + expect(runner).toHaveBeenCalledTimes(5); + for (const [, , args, options] of runner.mock.calls) { + expect(args).not.toContain("--admin-key"); + expect(args).not.toContain("--url"); + for (const form of deployKeyForms(key)) + expect(args.join(" ")).not.toContain(form); + expect(Object.keys(options.env!).sort()).toEqual([ + "CONVEX_DEPLOY_KEY", + "HOME", + "PATH", + "TMPDIR", + ]); + expect(options.env!.CONVEX_DEPLOY_KEY).toBe(key); + expect(options.secrets).toEqual(deployKeyForms(key)); + } + } finally { + runner.mockRestore(); + } + const root = join(import.meta.dirname, "../.."); + for (const file of ["harness/hosted.ts"]) + expect(await readFile(join(root, file), "utf8")).not.toContain( + "--admin-key", + ); +}); + +test("pure: a planted deploy key appears in no error, output, log line or record of a native run on a hosted deployment", async () => { + const key = plantedKey(); + const forms = deployKeyForms(key); + expect(forms).toHaveLength(4); + const echo = forms.join(" "); + const directory = await temporary("libar-hosted-planted-"); + // The stub CLI echoes every form of the key it was given, on both streams, and fails unless its + // working directory holds a file named succeed. + const script = join(directory, "stub-cli.mjs"); + await writeFile( + script, + [ + 'import { existsSync } from "node:fs";', + 'const key = process.env.CONVEX_DEPLOY_KEY ?? "";', + 'const secret = key.slice(key.indexOf("|") + 1);', + "const forms = [key, secret, encodeURIComponent(key), encodeURIComponent(secret)];", + 'console.log("stub ran: " + forms.join(" "));', + 'console.error("stub ran: " + forms.join(" "));', + 'process.exit(existsSync("succeed") ? 0 : 1);', + ].join("\n"), + ); + let functionCalls = 0; + const fetchStub: typeof globalThis.fetch = async (input) => { + const url = input instanceof Request ? input.url : String(input); + if (url.includes("/api/function")) { + functionCalls++; + if (functionCalls === 1) + return new Response( + JSON.stringify({ + status: "success", + value: null, + logLines: [`[LOG] ${echo}`], + }), + ); + if (functionCalls === 2) return new Response(echo, { status: 500 }); + return new Response(JSON.stringify({ status: "unexpected", echo })); + } + if ( + url.includes("/api/stream_function_logs") || + url.includes("/instance_version") + ) + throw new TypeError(`fetch failed: ${echo}`); + return new Response(echo, { status: 500 }); + }; + const socket: SocketFactory = (_url, onTransition): AdminSocketClient => ({ + setAdminAuth() {}, + subscribe() { + queueMicrotask(() => onTransition(["token"])); + return undefined; + }, + localQueryResultByToken() { + throw new Error(`the socket answered: ${echo}`); + }, + async close() {}, + }); + const composition = await temporaryComposition(); + const make = (signal?: AbortSignal) => + createHostedAccess({ + variables: variables(key), + composition, + home: directory, + state: { deployed: undefined, environment: new Set(), logProcess: {} }, + ...(signal === undefined ? {} : { signal }), + dependencies: { fetch: fetchStub, socket, cliScript: script }, + }).access; + const printed: string[] = []; + const capture = (...args: unknown[]) => { + printed.push(format(...args)); + }; + const spies = [ + ...(["log", "warn", "error", "debug", "info"] as const).map((method) => + vi.spyOn(console, method).mockImplementation(capture), + ), + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + printed.push(String(chunk)); + return true; + }), + vi.spyOn(process.stderr, "write").mockImplementation((chunk) => { + printed.push(String(chunk)); + return true; + }), + ]; + const failures: unknown[] = []; + const outputs: unknown[] = []; + let recordText = ""; + try { + const access = make(); + const failing = await temporary("libar-hosted-copy-"); + const succeeding = await temporary("libar-hosted-copy-"); + await writeFile(join(succeeding, "succeed"), ""); + const calls: (() => Promise)[] = [ + () => access.deploy(), + () => access.deployTemporary(failing), + () => access.codegen(), + () => access.setEnvironment({ NAME: "value" }), + () => access.environment(), + () => access.run("notes:add"), + () => access.run("notes:add"), + () => access.readTable("notes"), + () => access.writeTable("notes", { insert: { source: "x" } }), + () => + access.writeTable( + "streams", + { delete: "an-id" }, + { component: "depot" }, + ), + () => access.exportBackupArchive(join(directory, "archive.zip")), + () => access.importBackupArchive(join(directory, "archive.zip")), + async () => access.completionsSince(await access.logMark(), () => false), + () => access.usage(), + ]; + // The first function call succeeds and logs its log line through the client's logger. + outputs.push(await access.run("notes:add")); + outputs.push(await access.deployTemporary(succeeding)); + outputs.push(await access.instanceVersion()); + for (const call of calls) + await call().then( + (value) => outputs.push({ unexpected: value }), + (error: unknown) => failures.push(error), + ); + // A call that aborts rejects with its abort reason, redacted. + const controller = new AbortController(); + controller.abort(new Error(`aborted with ${echo}`)); + const aborted = make(controller.signal); + for (const call of [ + () => aborted.exportBackupArchive(join(directory, "archive.zip")), + () => aborted.deploy(), + ]) + await call().then( + (value) => outputs.push({ unexpected: value }), + (error: unknown) => failures.push(error), + ); + // The reporter writes the record of such a run, with an error that escaped redaction. + const runs = join(directory, "runs"); + await mkdir(runs); + const reporter = new EvidenceReporter({ directory: runs }); + const run = openRunRecord(); + run.secrets = forms; + run.hosted = { + ...hostedFacts, + usageBefore: { readAt: "now", seedStatus: null, response: { echo } }, + }; + reporter.onTestRunStart([{ project: { name: "hosted" } }] as never); + reporter.onTestCaseResult( + finished("hosted", { errors: forms.map((form) => `raw ${form}`) }), + ); + await reporter.onTestRunEnd([] as never, [] as never, "failed" as never); + amendRunRecord(run, (record) => record.unhandledErrors.push(echo)); + recordText = await readFile(run.path!, "utf8"); + } finally { + for (const spy of spies) spy.mockRestore(); + } + expect( + outputs.filter( + (output) => + typeof output === "object" && output !== null && "unexpected" in output, + ), + ).toEqual([]); + expect(failures).toHaveLength(16); + for (const failure of failures) + expect(String((failure as Error).message)).toContain("[redacted]"); + expect(JSON.stringify(outputs[1])).toContain("[redacted]"); + expect(printed.join("\n")).toContain("[redacted]"); + expect(recordText).toContain("[redacted]"); + const json = (value: unknown) => { + try { + return JSON.stringify(value); + } catch { + return String(value); + } + }; + const seen = [ + ...failures.flatMap((failure) => [ + String(failure), + (failure as Error).stack ?? "", + json(failure), + ]), + ...outputs.map(json), + ...printed, + recordText, + ]; + for (const form of forms) + expect( + seen.filter((text) => text.includes(form)), + "a form of the planted key appeared", + ).toEqual([]); +}); diff --git a/tests/pure/native.test.ts b/tests/pure/native.test.ts index e21b168..816c297 100644 --- a/tests/pure/native.test.ts +++ b/tests/pure/native.test.ts @@ -6,11 +6,15 @@ const context = vi.hoisted(() => ({ context: { signal: new AbortController().signal }, }, finished: [] as (() => Promise)[], + hostedRun: undefined as unknown, })); vi.mock("vitest", async (original) => ({ ...(await original()), TestRunner: { getCurrentTest: () => context.current }, - inject: () => ({ directory: "unused", executable: {} }), + inject: (key: string) => + key === "hostedRun" + ? context.hostedRun + : { directory: "unused", executable: {} }, onTestFinished: (finish: () => Promise) => context.finished.push(finish), })); @@ -19,7 +23,7 @@ vi.mock("../../harness/identity.js", () => ({ })); vi.mock("../../harness/backend.js", () => ({ startBackend: vi.fn() })); import { startBackend } from "../../harness/backend.js"; -import { fixtureBackend } from "../../harness/native.js"; +import { fixtureBackend, productionBackend } from "../../harness/native.js"; test("pure: a backend whose deploy fails is disposed and still contributes its facts to its test", async () => { context.current.meta = {}; @@ -92,3 +96,23 @@ test("pure: fixtureBackend owns an overridden executable and can skip deploy", a { composition: null }, ]); }); + +test("pure: fixtureBackend and productionBackend refuse a local backend under a native run on a hosted deployment", async () => { + context.current.meta = {}; + context.finished = []; + vi.mocked(startBackend).mockClear(); + context.hostedRun = { + target: { url: "https://happy-animal-123.convex.cloud" }, + startedAt: "2026-01-01T00:00:00.000Z", + }; + try { + for (const start of [fixtureBackend, productionBackend]) + await expect(start()).rejects.toThrow( + "fixtureBackend() and productionBackend() start a local backend, which a native run on a hosted deployment refuses", + ); + expect(startBackend).not.toHaveBeenCalled(); + expect(context.finished).toEqual([]); + } finally { + context.hostedRun = undefined; + } +}); From 2d5acdce91783bd2f05402151d37b197f2137958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:54:48 +0200 Subject: [PATCH 126/155] build: hosted the run setup and the record check harness/hosted-run.ts is the global setup of a hosted run: it selects the deployment from the four variables, refuses a composition directory with a dotenv file, reads the deployment's version and usage before the run and gives the reporter the secrets and the hosted facts; after the run it reads the usage again and amends the record in one step. scripts/hosted-record-check.mjs scans the captured log and the record for every form of the key it reads from its own environment, and names a file and a form's kind, never a value. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- harness/hosted-run.ts | 108 +++++++++++++++++++++++++++++ scripts/hosted-record-check.mjs | 45 ++++++++++++ tests/pure/hosted.test.ts | 118 +++++++++++++++++++++++++++++++- 4 files changed, 271 insertions(+), 2 deletions(-) create mode 100644 harness/hosted-run.ts create mode 100644 scripts/hosted-record-check.mjs diff --git a/design/README.md b/design/README.md index eb6a08c..2634e9a 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -237 specs · 5 packs · 133 anchors → 375 nodes · 1237 edges (0 errors, 0 warnings) +237 specs · 5 packs · 134 anchors → 376 nodes · 1238 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/harness/hosted-run.ts b/harness/hosted-run.ts new file mode 100644 index 0000000..778786a --- /dev/null +++ b/harness/hosted-run.ts @@ -0,0 +1,108 @@ +import { + codeAnchor, + codeAnchorId, + ref, +} from "@libar-dev/software-delivery-protocol"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { TestProject } from "vitest/node"; +import { refuseDotenv } from "./backup-archive.js"; +import { + compositions, + fixtureComposition, + projectDirectory, +} from "./composition.js"; +import { amendRunRecord, openRunRecord } from "./evidence.js"; +import type { HostedUsage, RunRecordSlot } from "./evidence.js"; +import { createHostedAccess, hostedTarget } from "./hosted.js"; +import type { HostedAccess } from "./hosted.js"; +const anchor = codeAnchor({ + id: codeAnchorId("impl:platform.native-harness.hosted-run"), + label: "the hosted run's setup and its amendment of the record", + satisfies: ref("spec:platform.native-harness"), +}); +void anchor; +// Whether a UTC day boundary falls between two times, which leaves a difference of two usage +// readings, each of the current UTC day, unsettled. +export function crossesUtcDay(from: string, to: string): boolean { + return from.slice(0, 10) !== to.slice(0, 10); +} +// A failed reading is recorded as none, with its message, already redacted by the access, on the +// run's standard error. +async function reading(access: HostedAccess): Promise { + try { + return await access.usage(); + } catch (error) { + process.stderr.write( + `The deployment's usage could not be read: ${(error as Error).message}\n`, + ); + return null; + } +} +async function cliVersion(): Promise { + const manifest = join( + import.meta.dirname, + "../node_modules/convex/package.json", + ); + return (JSON.parse(await readFile(manifest, "utf8")) as { version: string }) + .version; +} +// The global setup of npm run test:hosted. It selects the hosted deployment from the four process +// variables, refuses a composition directory with a dotenv file, reads the deployment's version +// and usage before the run, and gives the reporter the secrets and the hosted facts. After the run +// it reads the usage again and amends the record in one step. +export default async function setup( + project: TestProject, +): Promise<() => Promise> { + const variables = process.env; + const target = hostedTarget(variables); + for (const composition of compositions) + refuseDotenv(projectDirectory(composition)); + const startedAt = new Date().toISOString(); + const home = await mkdtemp(join(tmpdir(), "libar-hosted-run-")); + const { access, secrets } = createHostedAccess({ + variables, + composition: fixtureComposition, + home, + state: { deployed: undefined, environment: new Set(), logProcess: {} }, + }); + const run = openRunRecord(); + run.secrets = secrets; + run.hosted = { + deployment: target.deployment, + keyName: target.keyName, + statedPlan: target.statedPlan, + ranBy: + variables.GITHUB_ACTIONS === "true" + ? "continuous integration" + : "developer", + cliVersion: await cliVersion(), + backendVersion: await access.instanceVersion(), + usageBefore: await reading(access), + usageAfter: null, + windowCrossed: false, + }; + project.provide("hostedRun", { target, startedAt }); + return () => finishHostedRun(run, access, startedAt, home); +} +export async function finishHostedRun( + run: RunRecordSlot, + access: HostedAccess, + startedAt: string, + home: string, +): Promise { + try { + const usageAfter = await reading(access); + amendRunRecord(run, (record) => { + if (record.hosted === null) return; + record.hosted.usageAfter = usageAfter; + record.hosted.windowCrossed = crossesUtcDay( + record.hosted.usageBefore?.readAt ?? startedAt, + usageAfter?.readAt ?? new Date().toISOString(), + ); + }); + } finally { + await rm(home, { recursive: true, force: true }); + } +} diff --git a/scripts/hosted-record-check.mjs b/scripts/hosted-record-check.mjs new file mode 100644 index 0000000..a450463 --- /dev/null +++ b/scripts/hosted-record-check.mjs @@ -0,0 +1,45 @@ +// Scans what a native run on a hosted deployment leaves, its captured log and its record, for any +// form of the deploy key before the continuous integration prints or uploads them: +// `node scripts/hosted-record-check.mjs ...`. It reads the key from HOSTED_DEPLOY_KEY in its +// own environment and never prints it. The forms are those of harness/hosted.ts: the whole key, +// its part after the |, and both URL-encoded. +// Exit 0: no form in any file. 1: a form in a file, which it names with the form's kind and not its +// value. 2: it cannot check: no key, no file named, or a file it cannot read. +import { readFileSync } from "node:fs"; + +const key = process.env.HOSTED_DEPLOY_KEY ?? ""; +function cannot(reason) { + console.log(`hosted record check: cannot check · ${reason}`); + process.exit(2); +} +if (key === "") cannot("HOSTED_DEPLOY_KEY is not set"); +const files = process.argv.slice(2); +if (files.length === 0) cannot("no file is named"); +const secret = key.slice(key.indexOf("|") + 1); +const forms = [ + ["the whole key", key], + ["its part after the |", secret], + ["the whole key URL-encoded", encodeURIComponent(key)], + ["its part after the | URL-encoded", encodeURIComponent(secret)], +].filter(([, form]) => form !== ""); +let found = false; +for (const file of files) { + let text; + try { + text = readFileSync(file, "utf8"); + } catch (error) { + cannot(`${file} cannot be read: ${error.code ?? "unknown error"}`); + } + const held = forms.filter(([, form]) => text.includes(form)); + if (held.length === 0) continue; + found = true; + console.log( + `hosted record check: ${file} holds ${held.map(([kind]) => kind).join(", ")}`, + ); +} +console.log( + found + ? "hosted record check: a form of the deploy key was found" + : `hosted record check: no form of the deploy key in ${files.length} ${files.length === 1 ? "file" : "files"}`, +); +process.exit(found ? 1 : 0); diff --git a/tests/pure/hosted.test.ts b/tests/pure/hosted.test.ts index 05f2bfb..b50a466 100644 --- a/tests/pure/hosted.test.ts +++ b/tests/pure/hosted.test.ts @@ -3,6 +3,7 @@ import { specTest, testAnchorId, } from "@libar-dev/software-delivery-protocol"; +import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; import { mkdir, @@ -34,6 +35,7 @@ import { deployKeyForms, hostedTarget, } from "../../harness/hosted.js"; +import setupHostedRun, { crossesUtcDay } from "../../harness/hosted-run.js"; // Binds this file to the harness Spec, whose rules for a native run on a hosted deployment it // checks with no network and no deploy key. const anchor = specTest({ @@ -365,7 +367,7 @@ test("pure: no CLI call of the hosted path names the backend by an argument, and runner.mockRestore(); } const root = join(import.meta.dirname, "../.."); - for (const file of ["harness/hosted.ts"]) + for (const file of ["harness/hosted.ts", "harness/hosted-run.ts"]) expect(await readFile(join(root, file), "utf8")).not.toContain( "--admin-key", ); @@ -558,3 +560,117 @@ test("pure: a planted deploy key appears in no error, output, log line or record "a form of the planted key appeared", ).toEqual([]); }); + +test("pure: the run setup reads the version and the usage before the run, and after it amends the record in one step", async () => { + const key = plantedKey(); + const echo = deployKeyForms(key).join(" "); + for (const [name, value] of Object.entries(variables(key))) + vi.stubEnv(name, value); + vi.stubEnv("GITHUB_ACTIONS", "true"); + let readings = 0; + vi.stubGlobal("fetch", async (input: string | URL | Request) => { + const url = input instanceof Request ? input.url : String(input); + if (url.endsWith("/instance_version")) return new Response("1.29.0\n"); + if (url.endsWith("/api/v1/get_current_usage")) + return new Response( + JSON.stringify({ seedStatus: "complete", reading: ++readings, echo }), + ); + return new Response("unexpected", { status: 500 }); + }); + const provided: [string, unknown][] = []; + const directory = await temporary("libar-hosted-run-record-"); + try { + const teardown = await setupHostedRun({ + provide: (name: string, value: unknown) => provided.push([name, value]), + } as never); + expect(provided).toEqual([ + [ + "hostedRun", + { target: hostedTarget(variables(key)), startedAt: expect.any(String) }, + ], + ]); + const reporter = new EvidenceReporter({ directory }); + reporter.onTestRunStart([{ project: { name: "hosted" } }] as never); + reporter.onTestCaseResult(finished("hosted")); + await reporter.onTestRunEnd([] as never, [] as never, "passed" as never); + const [name] = await readdir(directory); + await teardown(); + const text = await readFile(join(directory, name!), "utf8"); + const record = JSON.parse(text) as NativeRunRecord; + expect(record.hosted).toMatchObject({ + deployment: "happy-animal-123", + keyName: "hosted-ci-scoped", + statedPlan: "Professional", + ranBy: "continuous integration", + cliVersion: "1.46.0", + backendVersion: "1.29.0", + usageBefore: { seedStatus: "complete", response: { reading: 1 } }, + usageAfter: { seedStatus: "complete", response: { reading: 2 } }, + windowCrossed: crossesUtcDay( + record.hosted!.usageBefore!.readAt, + record.hosted!.usageAfter!.readAt, + ), + deploys: [], + }); + for (const form of deployKeyForms(key)) expect(text).not.toContain(form); + expect(await readdir(directory)).toEqual([name]); + } finally { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); + } + expect( + crossesUtcDay("2026-10-05T23:59:59.000Z", "2026-10-06T00:00:01.000Z"), + ).toBe(true); + expect( + crossesUtcDay("2026-10-05T00:00:00.000Z", "2026-10-05T23:59:59.000Z"), + ).toBe(false); +}); + +function recordCheck(files: string[], key?: string) { + return new Promise<{ code: number; output: string }>((resolve) => + execFile( + process.execPath, + [ + join(import.meta.dirname, "../../scripts/hosted-record-check.mjs"), + ...files, + ], + { + env: { + PATH: process.env.PATH ?? "", + ...(key === undefined ? {} : { HOSTED_DEPLOY_KEY: key }), + }, + }, + (error, stdout) => + resolve({ + code: error === null ? 0 : Number(error.code), + output: stdout, + }), + ), + ); +} + +test("pure: the record check finds each form of the deploy key in a file, names the file and the form's kind, and prints no form", async () => { + const key = plantedKey(); + const directory = await temporary("libar-hosted-check-"); + const clean = join(directory, "clean.log"); + await writeFile(clean, "a run that printed no key\n[redacted]\n"); + expect(await recordCheck([clean], key)).toEqual({ + code: 0, + output: "hosted record check: no form of the deploy key in 1 file\n", + }); + for (const [index, form] of deployKeyForms(key).entries()) { + const file = join(directory, `form-${index}.json`); + await writeFile(file, `{"error": "x${form}x"}`); + const answer = await recordCheck([clean, file], key); + expect(answer.code).toBe(1); + expect(answer.output).toContain(`hosted record check: ${file} holds `); + expect(answer.output).not.toContain(clean + " holds"); + for (const any of deployKeyForms(key)) + expect(answer.output).not.toContain(any); + } + expect((await recordCheck([clean])).code).toBe(2); + expect((await recordCheck([], key)).code).toBe(2); + expect((await recordCheck([join(directory, "missing.json")], key)).code).toBe( + 2, + ); +}); From 8a80ae284cee5c804550306bfeb69d05567928b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:58:24 +0200 Subject: [PATCH 127/155] build: 4a audit and the diagnostic Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- .../specs/command/command-declaration.sdp.md | 2 +- design/specs/command/command-pipeline.sdp.md | 2 +- design/specs/command/outcome-boundary.sdp.md | 4 +- fixture/convex/orders.ts | 35 +++++- src/command/declaration.ts | 22 +++- src/command/index.ts | 3 +- src/command/pipeline.ts | 111 +++++++++++++++++- src/operations/diagnostic.ts | 109 +++++++++++++++++ src/operations/index.ts | 9 ++ src/read-model/write.ts | 7 +- 11 files changed, 282 insertions(+), 24 deletions(-) create mode 100644 src/operations/diagnostic.ts create mode 100644 src/operations/index.ts diff --git a/design/README.md b/design/README.md index 7659de1..5e49233 100644 --- a/design/README.md +++ b/design/README.md @@ -216,7 +216,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-35 | [`command.outcome-boundary`](specs/command/outcome-boundary.sdp.md) | The transient code `writePaused`, reserved for the write pause and confirmed by it | | E-36 | [`command.idempotency-and-receipts`](specs/command/idempotency-and-receipts.sdp.md) | The fingerprint is the SHA-256 digest of the canonical JSON, `convexToJson` with keys sorted, of the business input followed by the contract version; the outside facts the executor captures travel beside the input and are excluded, so a retry that captures a fresher fact is not a conflict | | E-37 | [`command.tenancy-and-authority`](specs/command/tenancy-and-authority.sdp.md) | Identity to actor mapping from `tokenIdentifier`, `subject` and `issuer`; grants written by the plain helpers `insertGrant` and `revokeGrant`, a tenant's first grant created with admin access through the composition's own internal grant mutation, with no grant function, bootstrap, first-admin flow or invitation in the library; revocation by deleting the grant row, with no cache; a 500-grant read bound | -| E-38 | [`command.command-declaration`](specs/command/command-declaration.sdp.md) | The declaration also carries `contractVersion`, `retention`, `irreversible`, `admission`, `bounds`, `rejections`, `audit` and `writes`; `bounds` is the one input bound, checked at the pipeline's step 1; `writes` names the context and stream type pairs the executor may write, from which step 7 derives the write pause's source scopes and which step 9 proves; the declaration has no reactions field | +| E-38 | [`command.command-declaration`](specs/command/command-declaration.sdp.md) | The declaration also carries `contractVersion`, `retention`, `irreversible`, `admission`, `bounds`, `rejections`, `audit`, `diagnosticSink` and `writes`; `bounds` is the one input bound, checked at the pipeline's step 1; `writes` names the context and stream type pairs the executor may write, from which step 7 derives the write pause's source scopes and which step 9 proves; the declaration has no reactions field | | E-39 | [`command.idempotency-and-receipts`](specs/command/idempotency-and-receipts.sdp.md) | The second UI submit of a create is answered with the rejection `entityExists`, not an idempotent success, because the UI path writes no receipt to replay from and the client holds the entity ID | | E-40 | [`application.rebuild`](specs/application/rebuild.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | A batch is one mutation with one cursor on the progress row, at most 100 entities capped by the row budget and by the context's byte-derived page bound, a design number not attributed to the component; a self-scheduled internal mutation drives every batch, the backfill, the verify pass and the purge, and the rebuild uses `@convex-dev/migrations` for nothing; one batch names one tenant, taken from the tenant list; a fence that every interrupt, resume, rollback, abort and purge start raises | | E-41 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | The caller-driven import command with an `imports` progress row; the input bound itself is the declaration's `bounds.maxItems` under E-38 | diff --git a/design/specs/command/command-declaration.sdp.md b/design/specs/command/command-declaration.sdp.md index 811ee31..1eea025 100644 --- a/design/specs/command/command-declaration.sdp.md +++ b/design/specs/command/command-declaration.sdp.md @@ -75,7 +75,7 @@ What generation would emit, once two modules show the pattern: for each declarat - generationTrigger: after two real modules; the generator is a deterministic script that emits the export pairs and the rejection-code union, and CI fails on a diff against the committed output (D12) - moduleGrouping: `class OrdersModule { static readonly commands = [placeOrderDeclaration, cancelOrderDeclaration] as const }` is allowed as grouping and holds no state the pipeline reads (D12) - canonicalSchema: `input` is a Convex validator and `refine` runs over its output; there is no second schema and no converter, so a refinement is never dropped silently (D12) -- refineResult: `refine` returns `null` when the validated input passes and otherwise the kernel's `Rejection` of `spec:kernel.outcome-model` without its `code`, a `message` and the failing path in `details`; the pipeline's step 1 throws that value through `reject` of `spec:command.outcome-boundary` with `code: "invalidInput"` and `commandType: decl.name`, so the refinement never chooses a code and never names the command (D12, D4, E-5) +- refineResult: `refine` returns `null` when the validated input passes and otherwise the kernel's `Rejection` of `spec:kernel.outcome-model` without its `code`, a `message` and the failing path in `details`; the pipeline's step 1 throws that value through `reject` of `spec:command.outcome-boundary` with `code: "invalidInput"` and `entry: decl.name`, so the refinement never chooses a code and never names the command (D12, D4, E-5) ## Example space diff --git a/design/specs/command/command-pipeline.sdp.md b/design/specs/command/command-pipeline.sdp.md index c9ea28e..3c987d5 100644 --- a/design/specs/command/command-pipeline.sdp.md +++ b/design/specs/command/command-pipeline.sdp.md @@ -77,7 +77,7 @@ The pipeline is one code path with two registered entries per command. The publi - rule: Read models updated in the pipeline are essential read models written from state that the same transaction committed; no invariant is checked against them (D8, Law 9) - rule: The mutation returns the outcome, the operation ID and the affected stream versions, so a client can compare what it reads against what it wrote (D4, D8, D2) - rule: [extension] Admission counts committed intent only: a consumption the policy writes rolls back with any later throw of the mutation, a rejection included, and an unauthorized call never reaches admission because step 4 precedes step 6; a policy that must bound refused traffic runs under the conditional dispatcher of `spec:command.outcome-boundary`, where the body is the nested mutation and the parent survives its rejection, never through a nested write inside this pipeline, whose writes commit only with the parent (E-32, D7, F3, Law 5) -- Step 1, parse: the entry's `args` validators run before the handler and refuse a request that does not fit the command's one canonical schema; after the public entry's step 2, which authenticates before the checks that follow, the handler checks every text field the caller set beside the input against its bound in UTF-8 bytes, in the order `limitCallText` lists them, throwing rejection `invalidInput` with `details` `{ field, length, limit }` for the first that is longer than its bound, then checks the declaration's name against `limitIdLength`, throwing a plain error for a name longer than it, because these fields become fields of the receipt row or of every event's envelope, then checks the declaration's `bounds`, throwing rejection `operationTooLarge` with the bound and the offered size when the input's items, the elements of its top-level arrays, exceed `bounds.maxItems` or its size measured with `getConvexSize` exceeds `bounds.maxBytes`, and runs the declaration's refinement over the validated input and, when it returns a value, throws rejection `invalidInput` carrying that value's `message` and `details` through `reject` of `spec:command.outcome-boundary` with the declaration's name as `commandType`, all before any read, so no executor body checks an input bound of its own (D12, D4, D10, E-34, E-38) +- Step 1, parse: the entry's `args` validators run before the handler and refuse a request that does not fit the command's one canonical schema; after the public entry's step 2, which authenticates before the checks that follow, the handler checks every text field the caller set beside the input against its bound in UTF-8 bytes, in the order `limitCallText` lists them, throwing rejection `invalidInput` with `details` `{ field, length, limit }` for the first that is longer than its bound, then checks the declaration's name against `limitIdLength`, throwing a plain error for a name longer than it, because these fields become fields of the receipt row or of every event's envelope, then checks the declaration's `bounds`, throwing rejection `operationTooLarge` with the bound and the offered size when the input's items, the elements of its top-level arrays, exceed `bounds.maxItems` or its size measured with `getConvexSize` exceeds `bounds.maxBytes`, and runs the declaration's refinement over the validated input and, when it returns a value, throws rejection `invalidInput` carrying that value's `message` and `details` through `reject` of `spec:command.outcome-boundary` with the declaration's name as `entry`, all before any read, so no executor body checks an input bound of its own (D12, D4, D10, E-34, E-38) - Step 2, authenticate: the public entry calls `establishActor` of `spec:command.actor-and-scope`, which reads `ctx.auth.getUserIdentity()` once, after Convex's argument validators and before step 1's key, bound and refinement checks, and throws rejection `unauthenticated` on `null`; the internal entry accepts the `actor` argument its trusted caller established and performs no authentication of its own (D11, F11) - Step 3, establish scope: the `tenantId` argument is the tenant scope for every read and write that follows; the pipeline resolves the actor's membership in that tenant through the grants read of step 4 and never infers a tenant from the identity (Law 11, D11) - Step 4, authorize: the pipeline reads the actor's grants for the tenant inside the transaction and evaluates the declaration's permission policy against actor, tenant scope and input; a denial throws rejection `forbidden` with `details` `{ reason }`, the reason `authorize` gave, before any execution and before any receipt is read (Law 5, D11, D12) diff --git a/design/specs/command/outcome-boundary.sdp.md b/design/specs/command/outcome-boundary.sdp.md index 6355c35..b2253fc 100644 --- a/design/specs/command/outcome-boundary.sdp.md +++ b/design/specs/command/outcome-boundary.sdp.md @@ -61,7 +61,7 @@ The closed code list has two halves. The platform codes below are closed here an ## Design -The kernel's `Outcome` maps one-to-one. Applied and business failure become the `CommandResponse` of the pipeline. Rejection becomes the rejection `ConvexError`. Technical failure is not a kernel outcome at all; it is whatever escapes as a plain throw. The transient kind is the pipeline's own: it comes from admission or from a maintenance gate, never from `decide`. The one catch on the public path is `normalizeThrown` at the top of the pipeline's handler: it rethrows every error, identifying rejection candidates and validating their shapes as `fnNormalizeThrown` states, then checking every candidate that fits, bare or already in the wire shape, against the platform codes, the declaration's `rejections`, the running command's name and the bound on details, wrapping a bare kernel `Rejection` that passes into `RejectionData`, rethrowing a rejection that fails as a plain error, and passing everything else through unchanged, so it never continues and never turns a technical failure into a rejection. +The kernel's `Outcome` maps one-to-one. Applied and business failure become the `CommandResponse` of the pipeline. Rejection becomes the rejection `ConvexError`. Technical failure is not a kernel outcome at all; it is whatever escapes as a plain throw. The transient kind is the pipeline's own: it comes from admission or from a maintenance gate, never from `decide`. The one catch on the public path sits at the top of the pipeline's handler and hands every error to `relayFailure` of `spec:command.command-pipeline`, which writes the failure's diagnostic record and throws what `normalizeThrown` throws: `normalizeThrown` rethrows every error, identifying rejection candidates and validating their shapes as `fnNormalizeThrown` states, then checking every candidate that fits, bare or already in the wire shape, against the platform codes, the declaration's `rejections`, the running command's name and the bound on details, wrapping a bare kernel `Rejection` that passes into `RejectionData`, rethrowing a rejection that fails as a plain error, and passing everything else through unchanged, so it never continues and never turns a technical failure into a rejection. The conditional dispatcher is the one place on the parent side where D7's nested mutation form appears. It runs the command's existing internal entry through `ctx.runMutation` so that a rejection rolls back only that nested body, then writes the refusal record through the normal command path of the context that owns that fact, and returns the rejection as a value. It costs one nested call per dispatched command and exists for no command until the product answers OQ2. @@ -77,7 +77,7 @@ The conditional dispatcher is the one place on the parent side where D7's nested - fnNormalizeThrown: [extension] `normalizeThrown(error: unknown, entry: string, rejections: readonly string[]): never` rethrows, where `entry` is the running entry's name: a `ConvexError` is a rejection candidate when its `data` is an object that is not an array and either has an own `code` property and no own `kind` property, the bare form, or has `kind` equal to `"rejection"`, the wire form; a candidate whose `data` does not fit its shape, the kernel's `rejectionValidator` for the bare form and the rejection branch of `errorDataValidator` for the wire form, is rethrown as a plain `Error` whose message names `entry`; to fit is to validate as Convex validates a value, so an array is not a record; every candidate that fits its shape, bare or in the wire shape, passes three checks in this order, and the first that fails rethrows a plain `Error` whose message names `entry` and what failed: first, `data.code` is a platform code or in `rejections`, and the message names the code; second, for a rejection in the wire shape, `data.entry` equals `entry`, and the message names the entry the rejection carried; third, `data.details` is absent or measures at most 16,384 bytes with `getConvexSize`, and the message names the measured byte count in decimal digits with no separator; a bare rejection that passes is rethrown as `new ConvexError({ kind: "rejection", entry, ...data })`, and a rejection in the wire shape that passes is rethrown as the same error object; a `ConvexError` carrying `kind` equal to `"transient"`, a `ConvexError` with any other data and every other thrown value are rethrown as the same value; it is called once, in the pipeline handler's outermost catch (D7, F14, Law 6, E-5) - fnReject: [extension] `reject(data: Omit, "kind">): never` throws `new ConvexError({ kind: "rejection", ...data })`, or a plain `Error` when `data.details` measure above 16,384 bytes with `getConvexSize`; `RejectionCode` is the platform half alone, so `data.code` is typed `PlatformRejectionCode` and a call with any other code does not compile; the helper throws the platform's own rejections, and a domain rejection is thrown as the kernel's bare `Rejection`, which the outermost catch checks against the declaration (D7, D12, E-5) - fnRefuseTransient: `refuseTransient(data: Omit): never` throws `new ConvexError({ kind: "transient", ...data })` (D4) -- fnClassify: `classifyThrown(error: unknown): { kind: "rejection"; data: RejectionData } | { kind: "transient"; data: TransientData } | { kind: "technical"; error: unknown }` used by tests, the dispatcher and the obligation wrapper, never by the public pipeline, which relays every throw; a `ConvexError` whose `data` does not fit `errorDataValidator`, a bare kernel `Rejection` included, is technical; to fit is to validate as Convex validates a value, as `fnNormalizeThrown` states, so `data` or `details` that are an array are not a record and such an error is technical (D7, Law 6) +- fnClassify: `classifyThrown(error: unknown): { kind: "rejection"; data: RejectionData } | { kind: "transient"; data: TransientData } | { kind: "technical"; error: unknown }` used by tests, the dispatcher, the obligation wrapper and the pipeline's `relayFailure`, which reads it only for the diagnostic record of what `normalizeThrown` threw and never to choose what it throws, so the public pipeline relays every throw; a `ConvexError` whose `data` does not fit `errorDataValidator`, a bare kernel `Rejection` included, is technical; to fit is to validate as Convex validates a value, as `fnNormalizeThrown` states, so `data` or `details` that are an array are not a record and such an error is technical (D7, Law 6) - errorCodeInvalidInput: [extension] thrown at step 1 with `details` `{ field, length, limit }` when a text field the caller set beside the input is longer than its bound in `limitCallText` of `spec:command.command-pipeline`, where `field` is the field's path as that list writes it, `length` its UTF-8 byte count and `limit` its bound; thrown by the persistence adapter at its step 1 with the same `details` and `field` equal to `"streamId"` when a planned stream ID is longer than 256 bytes; and thrown at step 1 when the declaration's refinement over validated input fails, carrying the failing path in `details` (D12, D4, D2, E-34, E-12) - errorCodeUnauthenticated: thrown at step 2 by the public entry, and by `authorizeQuery` in a parent query, when `ctx.auth.getUserIdentity()` returns `null` (D11) - errorCodeForbidden: thrown at step 4 with `details.reason`, the reason `authorize` gave, when the permission policy denies the actor for this command in this tenant, by `authorizeQuery` when the decision denies the actor the query's permission in the tenant, and on a duplicate whose caller no longer holds authorization, so nothing stored is disclosed (Law 5, Sc L1-8) diff --git a/fixture/convex/orders.ts b/fixture/convex/orders.ts index 8ffc053..9f255cb 100644 --- a/fixture/convex/orders.ts +++ b/fixture/convex/orders.ts @@ -1,17 +1,32 @@ // PlaceOrder, a fixture command whose use case makes one call to the depot's placeOrders operation, // which creates the order as a document under the client-generated order ID at expected version 0 // and then claims the order's stock lines. The create comes first, so a second submit of the same -// order is answered entityExists even when the first submit took the last units. +// order is answered entityExists even when the first submit took the last units. Its declaration +// carries the two faults of Sc ALL-1, keyed by data: an order whose audit record Convex refuses, and +// a tenant whose diagnostic lines the sink refuses. import { v, type Infer } from "convex/values"; import { internalCommand, publicCommand, type CommandDeclaration, } from "../../src/command/index.js"; +import { + consoleSink, + type DiagnosticSink, +} from "../../src/operations/index.js"; import { components } from "./_generated/api.js"; import { failBeforeReceiptIfSwitched, switchedAdmission } from "./switches.js"; const { operations } = components.depot; export const orderPermission = "depot.orders"; +// The order whose subject carries a field the auditRecords validator does not declare. +export const auditFaultOrderId = "order-audit-fault"; +// The tenant for which fixtureDiagnosticSink throws on every line. +export const sinkFaultTenant = "tenant-sink-fault"; +export const fixtureDiagnosticSink: DiagnosticSink = (line) => { + if (line.includes(`"tenantId":"${sinkFaultTenant}"`)) + throw new Error("Fault injected: the diagnostic sink is broken"); + consoleSink(line); +}; const stockLine = v.object({ productId: v.string(), quantity: v.number() }); const placeOrderInput = v.object({ orderId: v.string(), @@ -32,11 +47,17 @@ const placeOrderDeclaration: CommandDeclaration< output: placeOrderResult, permission: { permission: orderPermission, - subjectFrom: ({ orderId }) => ({ - contextId: "depot", - streamType: "document", - streamId: orderId, - }), + subjectFrom: ({ orderId }) => { + const subject = { + contextId: "depot", + streamType: "document", + streamId: orderId, + }; + if (orderId !== auditFaultOrderId) return subject; + // Authorization compares the three fields; Convex refuses the fourth at step 10's insert. + const refusedByAudit = { ...subject, auditFault: true }; + return refusedByAudit; + }, }, writes: [ { contextId: "depot", streamType: "document" }, @@ -45,6 +66,8 @@ const placeOrderDeclaration: CommandDeclaration< rejections: ["titleRequired", "insufficientStock", "invalidQuantity"], admission: switchedAdmission("PlaceOrder"), bounds: { maxItems: 100 }, + audit: { kind: "business" }, + diagnosticSink: fixtureDiagnosticSink, executor: async (ctx, { tenantId, actor, operation, input }) => { const placed = await ctx.runMutation(operations.placeOrders, { tenantId, diff --git a/src/command/declaration.ts b/src/command/declaration.ts index 6b9f9b0..8d28ec2 100644 --- a/src/command/declaration.ts +++ b/src/command/declaration.ts @@ -19,13 +19,15 @@ import { type SubjectRef, } from "./actor-and-scope.js"; import { establishActor } from "./authority.js"; -import { normalizeThrown, reject } from "./outcome-boundary.js"; +import { reject } from "./outcome-boundary.js"; import { commandResponseValidator, + relayFailure, runPipeline, type CommandResponse, type PipelineCall, } from "./pipeline.js"; +import type { DiagnosticSink } from "../operations/index.js"; import type { ReadModelBinding, SourceRef } from "../read-model/write.js"; import type { Retention } from "./receipts.js"; import type { CommandDataModel, MutationCtx } from "./tables.js"; @@ -48,6 +50,8 @@ export type CommandDeclaration = { retention?: Retention; // Asks step 10 for one audit record, which fails closed. Absent means none. audit?: { kind: "security" | "business" }; + // Where the pipeline writes the command's diagnostic lines. Absent means consoleSink. + diagnosticSink?: DiagnosticSink; }; export type PermissionPolicy = { permission: string; @@ -109,6 +113,7 @@ export function publicCommand( handler: async (ctx, args) => { const { tenantId, requestKey, correlationId, input } = args as PublicCommandArgs; + let established: Actor | undefined; try { // Step 2, authenticate. const actor = await establishActor(ctx, serviceIssuers); @@ -118,6 +123,7 @@ export function publicCommand( commandType: decl.name, message: `${decl.name} needs an authenticated caller`, }); + established = actor; return await runPipeline(ctx, decl, { tenantId, namespace: "public", @@ -127,7 +133,17 @@ export function publicCommand( ...(correlationId === undefined ? {} : { correlationId }), }); } catch (error) { - normalizeThrown(error, decl.name, decl.rejections); + relayFailure( + decl, + { + tenantId, + namespace: "public", + ...(established === undefined ? {} : { actor: established }), + ...(requestKey === undefined ? {} : { requestKey }), + ...(correlationId === undefined ? {} : { correlationId }), + }, + error, + ); } }, }) as RegisteredMutation<"public", PublicCommandArgs, CommandResponse>; @@ -152,7 +168,7 @@ export function internalCommand( try { return await runPipeline(ctx, decl, call); } catch (error) { - normalizeThrown(error, decl.name, decl.rejections); + relayFailure(decl, call, error); } }, }) as RegisteredMutation< diff --git a/src/command/index.ts b/src/command/index.ts index ee2b60b..a2b020c 100644 --- a/src/command/index.ts +++ b/src/command/index.ts @@ -63,9 +63,10 @@ export type { export { commandResponseValidator, outcomeKindValidator, + relayFailure, runPipeline, } from "./pipeline.js"; -export type { CommandResponse, PipelineCall } from "./pipeline.js"; +export type { CommandResponse, FailedCall, PipelineCall } from "./pipeline.js"; export { canonicalJson, classifyReceipt, diff --git a/src/command/pipeline.ts b/src/command/pipeline.ts index 8741551..2772f00 100644 --- a/src/command/pipeline.ts +++ b/src/command/pipeline.ts @@ -19,7 +19,12 @@ import type { AffectedRef, StreamVersion } from "../kernel/index.js"; import type { Actor, CallerNamespace } from "./actor-and-scope.js"; import { authorize } from "./authority.js"; import type { CommandDeclaration } from "./declaration.js"; -import { refuseTransient, reject } from "./outcome-boundary.js"; +import { + classifyThrown, + normalizeThrown, + refuseTransient, + reject, +} from "./outcome-boundary.js"; import { classifyReceipt, fingerprintOf, @@ -29,6 +34,11 @@ import { } from "./receipts.js"; import type { MutationCtx } from "./tables.js"; import { writeAudit, type AuditDataModel } from "../audit/index.js"; +import { + consoleSink, + emitDiagnostic, + type DiagnosticSink, +} from "../operations/index.js"; import { assertWritable, scopesOfUseCase } from "../gate/gate.js"; import type { GateDataModel } from "../gate/tables.js"; import { @@ -185,8 +195,66 @@ function parse(decl: CommandDeclaration, call: PipelineCall) { if (refused !== null) reject({ ...refused, code: "invalidInput", commandType }); } -// Steps 1 and 3 to 11 for either entry; the public entry ran step 2 before it built the call. Step 11 -// emits no diagnostic. +// What an entry has when it throws: actor is absent while the public entry has established none. +export type FailedCall = { + tenantId: string; + namespace: CallerNamespace; + actor?: Actor; + requestKey?: string; + correlationId?: string; + causedBy?: CausedBy; +}; +const sinkOf = (decl: CommandDeclaration): DiagnosticSink => + decl.diagnosticSink ?? consoleSink; +// The call's own fields of a diagnostic record, the same on every path. +function callFields(call: FailedCall) { + return { + tenantId: call.tenantId, + namespace: call.namespace, + ...(call.requestKey === undefined ? {} : { requestKey: call.requestKey }), + ...(call.correlationId === undefined + ? {} + : { correlationId: call.correlationId }), + ...(call.actor === undefined + ? {} + : { actorKind: call.actor.kind, actorId: call.actor.id }), + }; +} +// The whole body of each entry's outermost catch: the caller receives exactly what normalizeThrown +// alone would throw, after one diagnostic record of that throw. Nothing stored carries the operation +// ID, and nothing committed, so the record names no operation and counts no work. +export function relayFailure( + decl: CommandDeclaration, + call: FailedCall, + error: unknown, +): never { + let relayed: unknown = error; + try { + normalizeThrown(error, decl.name, decl.rejections); + } catch (thrown) { + relayed = thrown; + } + const classified = classifyThrown(relayed); + emitDiagnostic( + { + ...callFields(call), + commandType: decl.name, + kind: classified.kind, + ...(classified.kind === "technical" + ? {} + : { code: classified.data.code }), + replayed: false, + ...(call.causedBy === undefined ? {} : { causedBy: call.causedBy }), + eventsAppended: 0, + readModelRows: 0, + versions: [], + }, + sinkOf(decl), + ); + throw relayed; +} +// Steps 1 and 3 to 11 for either entry; the public entry ran step 2 before it built the call. A throw +// leaves the diagnostic record to the entry's relayFailure. export async function runPipeline( ctx: MutationCtx, decl: CommandDeclaration, @@ -248,6 +316,21 @@ export async function runPipeline( }); case "duplicate": { const stored = classified.receipt; + // The execution did no work: the receipt's operation and versions, and no counts. + emitDiagnostic( + { + ...callFields(call), + commandType, + kind: stored.outcome, + replayed: true, + operationId: stored.operationId, + ...(call.causedBy === undefined ? {} : { causedBy: call.causedBy }), + eventsAppended: 0, + readModelRows: 0, + versions: stored.versions, + }, + sinkOf(decl), + ); return { kind: stored.outcome, result: null, @@ -306,10 +389,11 @@ export async function runPipeline( throw new Error( `${commandType} wrote ${version.contextId}/${version.streamType}, which its declaration does not list in writes`, ); + let readModelRows = 0; if (decl.readModels !== undefined && decl.readModels.length > 0) // The library types ctx with the command tables. A parent that declares a read model holds the // registry and the read model's table too, and at run time ctx is that parent's own. - await writeReadModels( + readModelRows = await writeReadModels( ctx as unknown as RegistryReader & RowWriter, commandType, @@ -364,7 +448,24 @@ export async function runPipeline( }, ); } - // Step 11. + // Step 11: the diagnostic record from values the pipeline holds, then the outcome. + emitDiagnostic( + { + ...callFields(call), + commandType, + kind: executed.kind, + replayed: false, + operationId: operation.operationId, + causedBy: operation.causedBy, + eventsAppended: executed.streams.reduce( + (sum, { appended }) => sum + appended, + 0, + ), + readModelRows, + versions: executed.versions, + }, + sinkOf(decl), + ); return { kind: executed.kind, result: executed.result, diff --git a/src/operations/diagnostic.ts b/src/operations/diagnostic.ts new file mode 100644 index 0000000..c0f7d10 --- /dev/null +++ b/src/operations/diagnostic.ts @@ -0,0 +1,109 @@ +// The diagnostic record of spec:operations.baseline-operations: one bounded line per execution of a +// command entry's handler, written through a sink. emitDiagnostic never throws: an error the sink +// throws is swallowed and counted by one gap line, so no diagnostic failure reaches the mutation. +import type { ActorKind, CallerNamespace } from "../command/actor-and-scope.js"; +import type { CausedBy } from "../context/envelope.js"; +import { utf8Length } from "../context/text.js"; +import type { StreamVersion } from "../kernel/index.js"; +// kind is the outcome, written technical for a technical failure, or transient for a transient +// refusal, which is not an outcome, as classifyThrown names those two. +export type DiagnosticRecord = { + tenantId: string; + commandType: string; + namespace: CallerNamespace; + kind: "applied" | "businessFailure" | "rejection" | "transient" | "technical"; + code?: string; + replayed: boolean; + operationId?: string; + requestKey?: string; + correlationId?: string; + actorKind?: ActorKind; + actorId?: string; + causedBy?: CausedBy; + eventsAppended: number; + readModelRows: number; + versions: StreamVersion[]; +}; +export type DiagnosticSink = (line: string) => void; +export const consoleSink: DiagnosticSink = (line) => console.log(line); +// Bytes of UTF-8 per line diagnosticLine or diagnosticGapLine returns. +export const limitDiagnosticBytes = 4096; +const recordPrefix = "diagnostic "; +const gapPrefix = "diagnostic gap "; +// The keys a line holds, in the order of the record's type; versionsOmitted goes just before versions. +const recordKeys = [ + "tenantId", + "commandType", + "namespace", + "kind", + "code", + "replayed", + "operationId", + "requestKey", + "correlationId", + "actorKind", + "actorId", + "causedBy", + "eventsAppended", + "readModelRows", +] as const; +function recordLine( + record: DiagnosticRecord, + kept: number, +): { line: string; bytes: number } { + const fields: Record = {}; + for (const key of recordKeys) + if (record[key] !== undefined) fields[key] = record[key]; + const omitted = record.versions.length - kept; + if (omitted > 0) fields.versionsOmitted = omitted; + fields.versions = record.versions.slice(0, kept); + const line = recordPrefix + JSON.stringify(fields); + return { line, bytes: utf8Length(line) }; +} +// The longest prefix of whole code points that measures at most the bound. +function cutLine(line: string): string { + if (utf8Length(line) <= limitDiagnosticBytes) return line; + let bytes = 0; + let end = 0; + for (const point of line) { + const size = utf8Length(point); + if (bytes + size > limitDiagnosticBytes) break; + bytes += size; + end += point.length; + } + return line.slice(0, end); +} +export function diagnosticLine(record: DiagnosticRecord): string { + const whole = recordLine(record, record.versions.length); + if (whole.bytes <= limitDiagnosticBytes) return whole.line; + // Keep the longest prefix of versions for which the line, versionsOmitted counted, fits. + for (let kept = record.versions.length - 1; kept >= 0; kept--) { + const cut = recordLine(record, kept); + if (cut.bytes <= limitDiagnosticBytes) return cut.line; + } + return cutLine(recordLine(record, 0).line); +} +export function diagnosticGapLine(record: DiagnosticRecord): string { + const fields: Record = { + tenantId: record.tenantId, + commandType: record.commandType, + kind: record.kind, + }; + if (record.operationId !== undefined) fields.operationId = record.operationId; + return cutLine(gapPrefix + JSON.stringify(fields)); +} +// Reads and writes no table. Each error the sink throws, whatever is thrown, is counted by one gap line. +export function emitDiagnostic( + record: DiagnosticRecord, + sink: DiagnosticSink = consoleSink, +): void { + try { + sink(diagnosticLine(record)); + } catch { + try { + console.error(diagnosticGapLine(record)); + } catch { + // Nothing: a diagnostic failure never reaches the mutation. + } + } +} diff --git a/src/operations/index.ts b/src/operations/index.ts new file mode 100644 index 0000000..9444750 --- /dev/null +++ b/src/operations/index.ts @@ -0,0 +1,9 @@ +// The operations library: the diagnostic record a command writes beside its outcome. +export { + consoleSink, + diagnosticGapLine, + diagnosticLine, + emitDiagnostic, + limitDiagnosticBytes, +} from "./diagnostic.js"; +export type { DiagnosticRecord, DiagnosticSink } from "./diagnostic.js"; diff --git a/src/read-model/write.ts b/src/read-model/write.ts index 9577a7a..b8107e4 100644 --- a/src/read-model/write.ts +++ b/src/read-model/write.ts @@ -27,7 +27,8 @@ export const fromSource = ( entry.version.contextId === source.contextId && entry.version.streamType === source.streamType; // Every binding's read model must have a generation to write, whether or not an entry matches it. -// Returns the number of entries fed to the bindings. +// Returns the number of read-model rows inserted, replaced or deleted, a row in each generation once: +// the count held against limitReadModelWritesPerCommand, which the diagnostic record carries. export async function writeReadModels( ctx: RegistryReader & RowWriter, commandType: string, @@ -36,7 +37,6 @@ export async function writeReadModels( ): Promise { // One registry read per read model, however many sources bind it. const registry = new Map(); - let fed = 0; let rows = 0; for (const { readModel, source } of bindings) { let generations = registry.get(readModel.name); @@ -50,7 +50,6 @@ export async function writeReadModels( ); for (const entry of input.streams) { if (!fromSource(source, entry)) continue; - fed += 1; const results = await applyProjection(ctx, readModel, { tenantId: input.tenantId, dto: entry.dto, @@ -69,5 +68,5 @@ export async function writeReadModels( ); } } - return fed; + return rows; } From 9d8f079c2356b8d3733a6c0d065cbd7ad63093bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:58:31 +0200 Subject: [PATCH 128/155] build: hosted the copy's modules and the local example of the undeclared table tests/hosted/copy.ts writes the temporary copy of the fixture composition: the planting module and the table plantedSchedules of planting records in the parent, and the local in-place example's scheduler module in the parent and both fixture contexts. The native example on the local backend plants three schedules on the copy, deploys the production composition, whose schema leaves the table out, and the copy again, and finds the same three planting records. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- design/specs/platform/native-harness.sdp.md | 2 +- harness/child.ts | 3 +- tests/hosted/copy.ts | 97 +++++++++++++++++++ tests/hosted/planting.ts | 85 ++++++++++++++++ ...urvives-another-composition-deploy.test.ts | 75 ++++++++++++++ tests/pure/hosted.test.ts | 10 +- 7 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 tests/hosted/copy.ts create mode 100644 tests/hosted/planting.ts create mode 100644 tests/native/undeclared-table-survives-another-composition-deploy.test.ts diff --git a/design/README.md b/design/README.md index 2634e9a..7d73fa2 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -237 specs · 5 packs · 134 anchors → 376 nodes · 1238 edges (0 errors, 0 warnings) +237 specs · 5 packs · 135 anchors → 377 nodes · 1239 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index 3e5a39b..b4e957d 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -103,7 +103,7 @@ The harness is plain test support with no Convex function in it. The entries who - hostedConfiguration: [extension] `npm run test:hosted` runs `vitest run --config vitest.hosted.config.ts`, whose one project, `hosted`, runs one driver file under `tests/hosted/` with its own global setup and the evidence reporter; the driver runs four examples in order: Probe 7's retention example before any deploy or import, Probe 7's replacement in place, the production composition's acceptance, and Probe 3's quota half; the first, second and fourth run on one temporary copy of the fixture composition that adds the modules under `tests/hosted/`, and the fourth leaves that copy deployed for the next run; when the deployment holds no earlier planting, the first run deploys the copy before the retention example and records that it read nothing; `npm run test:all` never includes it (E-59) - hostedSchedule: [extension] the workflow `.github/workflows/hosted.yml` runs on `workflow_dispatch` and on a schedule at 03:00 UTC on Monday and Thursday; GitHub runs a schedule from the default branch, and the job runs in a GitHub environment named `hosted` that holds the key and admits the default branch only, so a dispatch from another branch gets no key; in one concurrency group that cancels no running job; it uploads the record as the artifact `hosted-evidence` only after `scripts/hosted-record-check.mjs`, reading the key from its own environment, finds no form of it in the record or the captured log, and no other workflow waits on it; until the workflow is on the default branch, the owner's machine runs `npm run test:hosted` with the variables in its shell, and its record names a developer as the one who ran it (E-59) - hostedRedactionTest: [extension] one pure test with no network plants a key in the form of a development deploy key, gives the hosted admin access an injected fetch and a stub socket and runs a stub CLI script, each of which echoes the key in all four forms on every path, calls every member so that each fails, captures `process.stdout` and `process.stderr` and the reporter's output, writes a record into a temporary directory, and fails when any form appears in any of them (E-59) -- hostedProbeCopy: [extension] the temporary copy of the fixture composition that the hosted driver deploys adds, from modules under `tests/hosted/`, the table `plantedSchedules` of planting records, one for each schedule a run planted in the parent, with its scheduler ID, the state and `completedTime` the run saw and the `startedAt` of the run that planted it, the mutation that plants one succeeded, one failed and one canceled schedule and writes their rows, the scheduler references and reactions of the local in-place example, and Probe 3's parent mutations; the fixture composition itself gains nothing, as `probe7Narrowed` of `spec:facts.probe-plan` requires, and a later run knows a schedule's age after its scheduler row is gone (E-59) +- hostedProbeCopy: [extension] the temporary copy of the fixture composition that the hosted driver deploys adds, from `tests/hosted/planting.ts` and the local in-place example's own scheduler module, the table `plantedSchedules` of planting records, one for each schedule a run planted in the parent, with its scheduler ID, the state and `completedTime` the run saw, the `startedAt` of the run that planted it and the count of backup archive imports since, the module `planting` that plants one succeeded, one failed and one canceled schedule and writes their planting records, and the scheduler references and reactions of the local in-place example in the parent and in the fixture contexts `depot` and `yard`; Probe 3's parent mutation is the fixture's own `readCost:viaMutation`, whose helper and component paths read the same document; the fixture composition itself gains nothing, as `probe7Narrowed` of `spec:facts.probe-plan` requires, and a later run knows a schedule's age after its scheduler row is gone (E-59) - compositionLayout: [extension] a composition is a name, a project directory and a functions directory; the Convex CLI reads `convex.json` and `package.json` from its working directory and takes no functions directory any other way, so the harness runs every CLI call in the composition's project directory; the fixture composition's project directory is the repository root, whose `convex.json` names `fixture/convex/`, and the production composition's is `example/`, whose `convex.json` names `convex/` and whose `package.json` is a symbolic link to the root's, so the repository has one `package.json` (E-15) - typeComposition: [extension] `interface Composition { name: "fixture" | "production"; project: string; functions: string; installedLayers: readonly string[] }` (E-15) - compositionTooling: [extension] code generation, the dev watch and the harness take a composition, and the type check, lint, format and the tests cover both; the generated files of both compositions are committed, the codegen script runs once per composition against a disposable backend, and CI fails when a committed generated file differs (E-15) diff --git a/harness/child.ts b/harness/child.ts index 3f81211..02423d5 100644 --- a/harness/child.ts +++ b/harness/child.ts @@ -98,7 +98,8 @@ export function redactingLogger(secrets: readonly string[]): ClientLogger { (...args: unknown[]) => console[method](redact(format(...args), secrets)); return { - logVerbose: write("debug"), + // Convex logs its verbose lines only when asked to, and the harness never asks. + logVerbose: () => undefined, log: write("log"), warn: write("warn"), error: write("error"), diff --git a/tests/hosted/copy.ts b/tests/hosted/copy.ts new file mode 100644 index 0000000..58ef216 --- /dev/null +++ b/tests/hosted/copy.ts @@ -0,0 +1,97 @@ +// The temporary copy of the fixture composition that the hosted driver deploys, and that the local +// example of an undeclared table deploys. It adds, from modules outside every composition, the +// table plantedSchedules of planting records and its planting module in the parent, and the +// scheduler references and reactions of the local in-place example in the parent and in the two +// fixture contexts. The fixture composition itself gains nothing. +import { createHash } from "node:crypto"; +import { + cp, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, onTestFinished } from "vitest"; +import type { Backend } from "../../harness/backend.js"; +import type { JsonValue } from "../../harness/evidence.js"; +const root = join(import.meta.dirname, "../.."); +// The parent and the fixture contexts whose sources the copy holds. The annex is mounted twice and +// the migrations component comes from a package, so neither gets a module. +export const copyScopes = [undefined, "depot", "yard"] as const; +export type CopyScope = (typeof copyScopes)[number]; +export const scopeOptions = (scope: CopyScope) => + scope === undefined ? {} : { component: scope }; +const schedulerTables = + 'schedulerData: defineTable({ value: v.string(), held: v.boolean() }), schedulerReferences: defineTable({ dispatchId: v.id("_scheduled_functions"), label: v.string() }), schedulerEffects: defineTable({ label: v.string(), value: v.string() }),'; +const plantingTable = + 'plantedSchedules: defineTable({ scheduledId: v.id("_scheduled_functions"), planted: v.string(), startedAt: v.string(), state: v.union(v.string(), v.null()), completedTime: v.union(v.number(), v.null()), imports: v.number() }),'; +const sha256 = (text: string) => + createHash("sha256").update(text).digest("hex"); +export interface HostedCopy { + directory: string; + // What the copy adds, recorded beside its deploy. + added: { [key: string]: JsonValue }; +} +// Writes the copy into a directory the test removes when it finishes. +export async function hostedCopy(): Promise { + const directory = await mkdtemp(join(tmpdir(), "libar-hosted-copy-")); + onTestFinished(() => rm(directory, { recursive: true, force: true })); + for (const path of ["fixture", "src", "package.json", "convex.json"]) + await cp(join(root, path), join(directory, path), { recursive: true }); + await symlink(join(root, "node_modules"), join(directory, "node_modules")); + const planting = await readFile( + join(import.meta.dirname, "planting.ts"), + "utf8", + ); + const scheduling = await readFile( + join(root, "tests/native/scheduler-functions.ts"), + "utf8", + ); + await writeFile(join(directory, "fixture/convex/planting.ts"), planting); + const schemas: { [scope: string]: string } = {}; + for (const scope of copyScopes) { + const path = join(directory, "fixture/convex", scope ?? ""); + await writeFile(join(path, "scheduling.ts"), scheduling); + let schema = await readFile(join(path, "schema.ts"), "utf8"); + if (!schema.includes('from "convex/values"')) + schema = 'import { v } from "convex/values";\n' + schema; + if (!schema.includes("defineTable")) + schema = schema.replace( + "{ defineSchema }", + "{ defineSchema, defineTable }", + ); + if (schema.split("defineSchema({").length !== 2) + throw new Error(`The schema of ${scope ?? "the parent"} changed shape`); + schema = schema.replace( + "defineSchema({", + `defineSchema({ ${schedulerTables}${scope === undefined ? plantingTable : ""}`, + ); + await writeFile(join(path, "schema.ts"), schema); + schemas[scope ?? "parent"] = sha256(schema); + } + return { + directory, + added: { + baseComposition: "fixture", + addedModules: ["planting", "scheduling"], + scopes: copyScopes.map((scope) => scope ?? "parent"), + plantingSha256: sha256(planting), + schedulingSha256: sha256(scheduling), + schemas, + }, + }; +} +// Plants one succeeded, one failed and one canceled schedule in the parent, with their planting +// records, and waits until each record holds its schedule's terminal state and completedTime. +export async function plantSchedules(backend: Backend, startedAt: string) { + await backend.admin.run("planting:plant", { startedAt }); + await expect + .poll(() => backend.admin.run("planting:settle", { startedAt }), { + timeout: 60000, + interval: 500, + }) + .toBe(true); +} diff --git a/tests/hosted/planting.ts b/tests/hosted/planting.ts new file mode 100644 index 0000000..a06f3e8 --- /dev/null +++ b/tests/hosted/planting.ts @@ -0,0 +1,85 @@ +// Deployed only in the hosted driver's temporary copy of the fixture composition, as the parent's +// module planting, and in the local example that a table's documents survive another +// composition's deploy. No composition ships it. +import { + internalMutationGeneric as mutation, + makeFunctionReference, +} from "convex/server"; +import type { GenericId } from "convex/values"; +import { v } from "convex/values"; +const reaction = makeFunctionReference<"mutation">("planting:react"); +const terminal = ["success", "failed", "canceled"]; +// What each planted schedule runs: it succeeds, or fails when it is told to. +export const react = mutation({ + args: { fail: v.boolean() }, + handler: (_ctx, { fail }) => { + if (fail) throw new Error("planted to fail"); + return null; + }, +}); +// Plants one schedule that succeeds, one that fails and one that is canceled, and writes a +// planting record for each, naming the run that planted it. +export const plant = mutation({ + args: { startedAt: v.string() }, + handler: async (ctx, { startedAt }) => { + const success = await ctx.scheduler.runAfter(0, reaction, { fail: false }); + const failed = await ctx.scheduler.runAfter(0, reaction, { fail: true }); + const canceled = await ctx.scheduler.runAfter(3600000, reaction, { + fail: false, + }); + await ctx.scheduler.cancel(canceled); + const planted = { success, failed, canceled }; + for (const [kind, scheduledId] of Object.entries(planted)) + await ctx.db.insert("plantedSchedules", { + scheduledId, + planted: kind, + startedAt, + state: null, + completedTime: null, + imports: 0, + }); + return planted; + }, +}); +// Copies into each planting record of a run the state and completedTime its schedule shows once +// the schedule is terminal, so that a later run knows the schedule's age after its row is gone. +// Answers whether every record of the run is settled. +export const settle = mutation({ + args: { startedAt: v.string() }, + handler: async (ctx, { startedAt }) => { + let settled = true; + for (const record of await ctx.db.query("plantedSchedules").collect()) { + if (record.startedAt !== startedAt || record.completedTime !== null) + continue; + const scheduled = await ctx.db.system.get( + record.scheduledId as GenericId<"_scheduled_functions">, + ); + const kind = scheduled?.state.kind; + if ( + scheduled === null || + scheduled.completedTime === undefined || + kind === undefined || + !terminal.includes(kind) + ) { + settled = false; + continue; + } + await ctx.db.patch("plantedSchedules", record._id, { + state: kind, + completedTime: scheduled.completedTime, + }); + } + return settled; + }, +}); +// Counts one backup archive import in every planting record the import restored. +export const countImport = mutation({ + args: {}, + handler: async (ctx) => { + for (const record of await ctx.db.query("plantedSchedules").collect()) + await ctx.db.patch("plantedSchedules", record._id, { + imports: Number(record.imports) + 1, + }); + return null; + }, +}); diff --git a/tests/native/undeclared-table-survives-another-composition-deploy.test.ts b/tests/native/undeclared-table-survives-another-composition-deploy.test.ts new file mode 100644 index 0000000..8faa898 --- /dev/null +++ b/tests/native/undeclared-table-survives-another-composition-deploy.test.ts @@ -0,0 +1,75 @@ +import { isDeepStrictEqual } from "node:util"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import type { Value } from "convex/values"; +import { expect, vi } from "vitest"; +import { undeclaredTableSurvivesAnotherCompositionDeployContract as contract } from "../../generated/contracts/platform.native-harness.undeclared-table-survives-another-composition-deploy.contract.js"; +import type { Backend } from "../../harness/backend.js"; +import { measure, productionBackend, required } from "../../harness/native.js"; +import { hostedCopy, plantSchedules } from "../hosted/copy.js"; +const anchor = specTest({ + id: testAnchorId( + "test:platform.native-harness.undeclared-table-survives-another-composition-deploy", + ), + verifies: ref( + "spec:platform.native-harness.undeclared-table-survives-another-composition-deploy", + ), +}); +void anchor; +vi.setConfig({ testTimeout: 300000 }); +interface World { + backend?: Backend; + copy?: string; + planted?: Record[]; + deployError?: string; + after?: Record[]; +} +bindExample(contract, (): World => ({}), { + "a local backend running a temporary copy of the fixture composition whose schema declares the table `plantedSchedules`, holding {rows} planting records": + async (world, { rows }) => { + const backend = await productionBackend({ deploy: false }); + const copy = await hostedCopy(); + measure("temporary deployment", { + ...copy.added, + ...(await backend.admin.deployTemporary(copy.directory)), + }); + await plantSchedules(backend, new Date().toISOString()); + world.planted = await backend.admin.readTable("plantedSchedules"); + expect(world.planted).toHaveLength(rows); + Object.assign(world, { backend, copy: copy.directory }); + }, + "the harness deploys the production composition, whose schema leaves that table out, and then deploys the same temporary copy again": + async (world) => { + const backend = required(world.backend, "the backend"); + try { + await backend.admin.deploy(); + } catch (error) { + world.deployError = String(error); + } + // What an admin read finds while the table is undeclared is recorded, not bound. + measure( + "planting records while the production composition is deployed", + await backend.admin.readTable("plantedSchedules").then( + (rows) => rows.length, + (error: unknown) => String(error), + ), + ); + await backend.admin.deployTemporary(required(world.copy, "the copy")); + world.after = await backend.admin.readTable("plantedSchedules"); + }, + "the production composition's deploy completes {deployed}": ( + world, + { deployed }, + ) => { + expect(world.deployError === undefined, world.deployError).toBe(deployed); + }, + "the table holds the same planting records {rows} with their IDs and creation times {unchanged}": + (world, { rows, unchanged }) => { + expect(world.after).toHaveLength(rows); + expect(isDeepStrictEqual(world.after, world.planted)).toBe(unchanged); + }, +}); diff --git a/tests/pure/hosted.test.ts b/tests/pure/hosted.test.ts index b50a466..14b0038 100644 --- a/tests/pure/hosted.test.ts +++ b/tests/pure/hosted.test.ts @@ -367,7 +367,15 @@ test("pure: no CLI call of the hosted path names the backend by an argument, and runner.mockRestore(); } const root = join(import.meta.dirname, "../.."); - for (const file of ["harness/hosted.ts", "harness/hosted-run.ts"]) + const hostedFiles = [ + "harness/hosted.ts", + "harness/hosted-run.ts", + ...(await readdir(join(root, "tests/hosted"))).map( + (name) => `tests/hosted/${name}`, + ), + ]; + expect(hostedFiles.length).toBeGreaterThan(2); + for (const file of hostedFiles) expect(await readFile(join(root, file), "utf8")).not.toContain( "--admin-key", ); From 71ac95ed3e48f5f0f632349200c133d20f3504e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:58:40 +0200 Subject: [PATCH 129/155] design: the close of round 3, ten history findings fixed or partially fixed on paper Co-Authored-By: Claude Opus 5.5 --- design/reviews/consensus-ledger.json | 44 +++++++++++++++------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index 20230c0..025b380 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -1934,8 +1934,8 @@ ], "issue": "A `HistoryProjection` folds events of several streams into one row, and the two paths feed it in different orders. On the live path step 9 calls `applyHistoryProjection` per `streams` entry in the executor's order, command after command, so an order's row sees `OrderPlaced`, its `StockAllocated` events, later `OrderCancelled`, then `AllocationReleased`. On the backfill the rebuild pages one stream's whole history at a time (rebuild flow 'each history batch takes one stream from the enumeration'), so the same row sees every Orders event first and the stock events after, or the reverse. The contract asks only that `project` be 'deterministic in the prior row and the events' with 'version order within each stream'. It never says the fold must give the same row whatever the interleaving across streams, and D2 gives no order across streams to appeal to. A natural projection, where `OrderPlaced` creates the row and a stock event updates an existing one, backfills to a different row than it wrote live, against the contract's own value line that 'a rebuild cannot disagree with a live update'. The doc sends a consumer whose result depends on order to Layer 6's ordered consumer; Layer 2 does not say its history view must not be one. Deletion makes it worse: `null` from `project` deletes the row, and with it the `sourceVersions` that are the fold's only memory. A later event of the same key from another stream, such as `AllocationReleased` after `OrderCancelled` returned `null`, meets no prior row and recreates one, on either path.", "fix": "Add to projection-contract under E-9: a history projection's fold is independent of the order in which different streams' events arrive; only the order within one stream is given (D2), and a view whose meaning depends on cross-stream order waits for the ordered consumer of D18. Add to `determinismCheck` a test that folds one key's events in two interleavings and compares the rows, and have the L2-6 example compare a backfilled row with the live-written one. Rule on deletion: either a history row is never deleted once written and carries its own ended state, or the highest folded version per stream moves off the row into a marker, like `projectionMarkers`, that outlives it. State in orders-inventory-example that the order allocation history fold is written to hold under any interleaving, including a stock event arriving before `OrderPlaced`.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-04: At HEAD, projection-contract.sdp.md:71 promises \"version order within each stream\" and says null deletes the row, while :72 keeps sourceVersions only on that row, so stock-first history or a late event after deletion can disagree with live folding. The issue and interleaving-independent fold plus persistent deduplication fix remain right; define baseline handling too. The build must show: Pure tests must compare legal cross-stream interleavings and page boundaries, including stock before order events; convex-test must preserve deduplication through deletion, late input and repeated pages, with explicit baseline handling.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-04: At HEAD, projection-contract.sdp.md:71 promises \"version order within each stream\" and says null deletes the row, while :72 keeps sourceVersions only on that row, so stock-first history or a late event after deletion can disagree with live folding. The issue and interleaving-independent fold plus persistent deduplication fix remain right; define baseline handling too. The build must show: Pure tests must compare legal cross-stream interleavings and page boundaries, including stock before order events; convex-test must preserve deduplication through deletion, late input and repeated pages, with explicit baseline handling. Close of round 3, 4c, on paper: projection-contract promises one history row whatever the order across streams and however events are split between calls, each field written by one stream; a history projection's project never answers null, so a row keeps its ended state until purge; a pure test over 90 interleavings and every split (tests/pure/order-allocation-history.test.ts). The convex-test and native proof are the history view's build.", "slice": "S4", "detector": "pure unit test" }, @@ -1951,8 +1951,8 @@ ], "issue": "Two parts of the paused rebuild cannot be built as pinned. First, a history view has several sources, the order stream type and the stock item stream type in the example, but `batchCursorValidator` is `{ tenantId, pageCursor, streamId?, eventCursor? }` with no field for the source being enumerated, and `fnHistoryPageCall` names one component query. `historyStep3` says 'it calls `list` with `numItems` 1 from the page cursor to take the next stream' without saying of which context and stream type, so the batch cannot tell where it is among the bindings. Second, `historyVerify` checks one thing per stream: it reads the stream's last event, 'finds that event's row by `keyOf`', and counts a miss when the row is absent or records a lower version. `keyOf` may return `null`, and for many streams the last event's key is `null`: a stock item that was received and never allocated, and every stream whose last event is a `baseline` after a meaning migration. The Spec does not say what verify does then. Read as a miss, the repair refolds the stream, changes nothing, and the next pass counts the same miss, so the generation never reaches `verified` and, since 'writes reopen only after verification', the sources stay closed until an operator aborts. Read as a skip, the stream is unverified. Either way the check proves only that the last event of a stream was folded, not the earlier ones that belong to other keys.", "fix": "Add the source to the cursor, an index into the read model's source list, and iterate tenant, then source, then stream, then event page; say so in `tenantIteration` and `historyStep3`. Replace `historyVerify` with a second pass of the same history batches that counts the events `applyHistoryProjection` did not drop: under the pause nothing changes, so a covered generation folds zero events and a miss is repaired by the pass that found it. The state moves to `verified` when a full pass folds none. That needs no row lookup by key and handles a `null` key by construction. State the cost, one more read of every source's history under the pause.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-03: At HEAD, generation-registry.sdp.md:62 stores tenant/page/stream/event without a source binding and rebuild.sdp.md:117 checks only \"the stream's last event\", so stock contributions to A at v2 and B at v3 can pass with A missing. The source-aware cursor fix remains right, but a zero-new-fold verification pass needs proof that higher watermarks cannot hide holes and must define null-key and baseline treatment. The build must show: convex-test must interrupt at each source, tenant, stream and event-page position across two contexts with equal local IDs, stock contributing to orders A and B, unallocated stock, trailing stock receipts and baselines. Deleting A's earlier row must be detected and repaired, followed by native backend interruption/resume under the pause without missing relevant rows or looping on a null key.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-03: At HEAD, generation-registry.sdp.md:62 stores tenant/page/stream/event without a source binding and rebuild.sdp.md:117 checks only \"the stream's last event\", so stock contributions to A at v2 and B at v3 can pass with A missing. The source-aware cursor fix remains right, but a zero-new-fold verification pass needs proof that higher watermarks cannot hide holes and must define null-key and baseline treatment. The build must show: convex-test must interrupt at each source, tenant, stream and event-page position across two contexts with equal local IDs, stock contributing to orders A and B, unallocated stock, trailing stock receipts and baselines. Deleting A's earlier row must be detected and repaired, followed by native backend interruption/resume under the pause without missing relevant rows or looping on a null key. Close of round 3, 4c, on paper: the sourceIndex cursor orders a pass by tenant, source, stream and event page, and verification repeats the backfill and counts each row inserted or updated as a miss. The proof is the history view's build.", "slice": "S4", "detector": "convex-test" }, @@ -1969,7 +1969,7 @@ "issue": "`rollbackGeneration` takes any read model: it sets the retired generation to `verifying`, 'live commands write it again from that commit', and a verify pass repairs it. For a history view nothing closes the sources: write-pause says no function other than `startGeneration` closes a source scope for a rebuild. So after a rollback the next live command folds its new events onto a retired row that never saw the events between the switch and the rollback, and `applyHistoryProjection` raises that row's `sourceVersions` to the new event's version. The missed events are now 'not above the recorded version' and are dropped by every later fold, and `historyVerify`, which only compares the last event's version, passes. The rolled-back generation becomes active with holes in it and no check says so. projection-contract's claim that 'the live path never meets a non-active generation of a history view because its building generation exists only while its source scopes are closed' is false on this path, and rebuild's own `rebuildClass` bullet says why it matters: 'a live fold onto a row backfill has not reached cannot be told apart from a complete one'.", "fix": "Rule one way under E-8. Either `rollbackGeneration` refuses a `pauseRequired` generation with a plain error, and rolling a history view back means a new paused generation at the old projection version; or it closes every source scope of the view in the same mutation, naming the reopened generation, runs its catch-up pass under the pause, and the scopes reopen only after the switch, exactly as a first build does. Correct the projection-contract sentence to match, and add the rollback of the history view, or its refusal, to the L2-6 examples.", "status": "partially-fixed", - "note": "the platform review of 2026-10-02, APPL-05: At HEAD, rebuild.sdp.md:108 sets a retired generation to \"verifying\" without closing its sources, while projection-contract.sdp.md:72 drops events below the row watermark, so v10 can hide missed v7-v9. The issue and refusing rollback or reacquiring the write pause remain right. The build must show: convex-test must retire at v6, commit v7-v9 and race v10 with rollback, then repeat on the native backend; all relevant effects must be present or v10 refused while the required pause remains held. S4 on paper, 2026-10-02. `spec:application.rebuild` `fnRollbackGeneration` refuses a generation with `pauseRequired`, and the E-8 question on what the workflow does not offer says why. `spec:application.rebuild.write-pause-rebuild-interrupt` still binds 'can roll back', for roadmap 4c, which writes the paused path; the build must show the refusal at convex-test.", + "note": "the platform review of 2026-10-02, APPL-05: At HEAD, rebuild.sdp.md:108 sets a retired generation to \"verifying\" without closing its sources, while projection-contract.sdp.md:72 drops events below the row watermark, so v10 can hide missed v7-v9. The issue and refusing rollback or reacquiring the write pause remain right. The build must show: convex-test must retire at v6, commit v7-v9 and race v10 with rollback, then repeat on the native backend; all relevant effects must be present or v10 refused while the required pause remains held. S4 on paper, 2026-10-02. `spec:application.rebuild` `fnRollbackGeneration` refuses a generation with `pauseRequired`, and the E-8 question on what the workflow does not offer says why. `spec:application.rebuild.write-pause-rebuild-interrupt` still binds 'can roll back', for roadmap 4c, which writes the paused path; the build must show the refusal at convex-test. Close of round 3, 4c, on paper: a rollback of a paused view is refused on every path (rebuild, generation-registry, write-pause, projection-contract), and the L2-6 interrupt example binds the refusal. The proof is the history view's build.", "slice": "S4", "detector": "convex-test" }, @@ -1985,10 +1985,11 @@ ], "issue": "The unfixed half of r2-architecture-history-and-cross-context-projection-has-no-input. The contract still says a cross-context row 'holds one `StreamVersion` per context' and that such a row spreads `rowConventions`, read-models offers 'one cross-context read-model row' as a read, and the declaration says 'a read model fed by several stream types is bound once per source with the same projection'. For a `Projection⟨D, Row⟩` that cannot work: `project(tenantId, dto, versions)` builds a whole row from one DTO, and `applyProjection` writes it with `ctx.db.insert` or `ctx.db.replace`. Bound to two sources, the second entry's row replaces the first, so the row holds one context's fields and one context's version. No form takes two DTOs or merges into a prior row, and the backfill enumerates one source. The only form that can hold two contexts is `HistoryProjection`, which rebuilds only under the write pause, although D9 says a current-state read model rebuilds online. An implementer who wants D8's 'small cross-context view' as a read model has to guess between a history view with a pause it does not need and a form the corpus does not define.", "fix": "Take one of the two options the round-2 finding gave. Narrow: state in projection-contract and read-models that Layer 2 pins per-entity, aggregate and history forms, that a small cross-context current-state view is D8's parent query over bounded component reads, and remove 'cross-context row' from the `Projection` bullets, the row conventions and `fnCrossContextView`; record the missing composite form as a [non-blocking] E-9 question. Or add `CompositeProjection`: `project` takes the DTOs of every matching `streams` entry of one command plus the prior row, the write compares `sourceVersions` per stream, and the backfill composes one `list` DTO with bounded `get` reads of the other context in the same mutation, online.", - "status": "open", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: A read-model row needs current DTOs from two contexts, but the declaration and live writer only pass one stream entry at a time; the unsupported generic promise has no complete write form. The build must show: Compiled contracts and convex-test must either provide the complete source-read/write form and compare live output with rebuild after either source changes, or explicitly narrow the supported promise to the per-subject path until an actual consumer requires it. S4 on paper, 2026-10-02. Narrowed in `spec:application.rebuild`, the E-8 question on what the workflow does not offer: a row built from more than one source stream type has no rebuild there. Its write form on the live path, in `spec:application.projection-contract` and `spec:application.read-models`, is not designed.", + "status": "fixed", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: A read-model row needs current DTOs from two contexts, but the declaration and live writer only pass one stream entry at a time; the unsupported generic promise has no complete write form. The build must show: Compiled contracts and convex-test must either provide the complete source-read/write form and compare live output with rebuild after either source changes, or explicitly narrow the supported promise to the per-subject path until an actual consumer requires it. S4 on paper, 2026-10-02. Narrowed in `spec:application.rebuild`, the E-8 question on what the workflow does not offer: a row built from more than one source stream type has no rebuild there. Its write form on the live path, in `spec:application.projection-contract` and `spec:application.read-models`, is not designed. Close of round 3, 4c: the cross-context row is narrowed to D8's parent query in read-models and projection-contract; the missing composite form is an E-9 question.", "slice": "S4", - "detector": "convex-test" + "detector": "convex-test", + "unreviewed": true }, { "round": 3, @@ -2113,8 +2114,8 @@ ], "issue": "Step 9 calls `applyHistoryProjection` 'for each `streams` entry', and that helper reads and replaces the row once per key per call. `PlaceOrder` at 100 lines returns 101 entries that all key to the same order, so the order allocation history row is read and replaced 101 times in one mutation, about 200 operations and up to 3 MiB written for one row. The cost is recorded nowhere: `limitHistoryFold` counts rows per entry, E-45's four read-model rows per command is silent on repeated writes, and `limitExpectedDocuments` omits the view. The row itself, 100 lines plus 101 `sourceVersions` entries, is over the default `rowBudgetBytes` of 16 KiB and the table bullet declares no budget. And the Specs do not say whether the measured `PlaceOrder` binds the view: it 'exists in the experiment for Sc L2-6, not as an essential read', yet a binding on the declaration is written by every command.", "fix": "Have step 9 gather the events of every matching entry of one command per history projection and call `applyHistoryProjection` once, so each key costs one read and one write; say so in `limitHistoryFold`. Declare `rowBudgetBytes` for `orderAllocationHistory`, 64 KiB at the provisional maximum. State in first-experiment whether the Sc L2-3 runs measure `PlaceOrder` with the view bound, and add its row to `limitExpectedDocuments` if they do.", - "status": "open", - "note": "S2, 2026-10-02, fold-in after the build. The read-model write limit of 4 counts the rows step 9 writes, not the `streams` entries its bindings match (ruling C3, `spec:application.read-models` and `spec:command.command-pipeline`), checked after each entry is applied; a history binding fed by 101 entries that key to one row counts that row's writes, not the entries.", + "status": "partially-fixed", + "note": "S2, 2026-10-02, fold-in after the build. The read-model write limit of 4 counts the rows step 9 writes, not the `streams` entries its bindings match (ruling C3, `spec:application.read-models` and `spec:command.command-pipeline`), checked after each entry is applied; a history binding fed by 101 entries that key to one row counts that row's writes, not the entries. Close of round 3, 4c, on paper: step 9 calls applyHistoryProjection once per command for each history read model, the view declares a 64 KiB row budget, and first-experiment states the added document written and read. The step 9 wording in command-pipeline lands after the audit lane's merge.", "slice": "S4", "detector": "judgement" }, @@ -2128,8 +2129,8 @@ ], "issue": "Under the pause 'each history batch takes one stream from the enumeration' and reads one `history` page of it, so a stream with two events costs a whole scheduled mutation, a `list` call of one item and a `history` call. The pause closes the view's sources for every tenant of the deployment, because a generation spans all tenants, so the outage of those writers lasts as long as the number of source streams times one batch. `limitHistoryBatch` bounds events per batch at 128 by default and the batch uses a fraction of it for short streams. The duration is an unrecorded cost of the mechanism, and a cheaper batch passes the same scenario.", "fix": "Let a history batch take streams from the enumeration until its event bound is spent, with the stream and event cursor saved as now, and state the pause's expected duration as a risk on rebuild: source streams and events divided by the batch bound, times one mutation. Point at the registry's per-tenant alternative as the way to shorten the outage per tenant.", - "status": "open", - "note": "", + "status": "partially-fixed", + "note": "Close of round 3, 4c: one stream per batch is kept, the do-nothing option, and the pause length is stated as a risk in rebuild; a batch of several streams needs a second bound and is not taken.", "slice": "S4", "detector": "judgement" }, @@ -2590,8 +2591,8 @@ ], "issue": "The round-2 fix r2-completeness-aggregate-row-shape-unpinned pinned `applyAggregate` in three steps that are wrong when an entity moves from one aggregate key to another, which is the case the Spec's own sample table is built for: `orderTotals` is 'an aggregate of order totals per status, where `key` is the status'. aggregateStep2 computes 'the difference is `contribution` minus the marker's recorded contribution'. aggregateStep3 'reads the aggregate row by `by_key` with `aggregateKeyOf(tenantId, dto)`, inserts it with `value` equal to the difference and `entityCount` 1 when missing, else adds the difference to `value` and adjusts `entityCount` by one for a new marker and by minus one for a removed one'. When an order goes from placed to cancelled its marker exists with `aggregateKey` placed and the same contribution, so the difference is zero: the cancelled row gains nothing and no count, or is inserted with `value` 0 and `entityCount` 1, and the placed row keeps the order's total and count for ever. The marker stores `aggregateKey`, and no step reads it. Two smaller holes sit in the same steps: a missing aggregate row is inserted with `entityCount` 1 even when the contribution is `null`, and the input's `entityKey` has no stated source, since `AggregateProjection` has no `entityKeyOf` and step 9 is said to call the helper 'with the entry's DTO and version'.", "fix": "In `spec:application.projection-contract` split aggregateStep3 on the marker's `aggregateKey`: when it differs from `aggregateKeyOf(tenantId, dto)`, read the old key's row, subtract the recorded contribution and one from `entityCount`, then add the whole new contribution and one to the new key's row; when it is the same, add the difference as now; never insert a row for a `null` contribution. State the cost, a second row read and write on a key change, and count it against E-45's four rows per command. Pin `entityKey` as the entry's stream identity, `streamType` and `streamId` from `dto.version`, in `fnApplyAggregate` and in the pipeline's step 9. Add a key change to `determinismCheck`.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-02: At HEAD, projection-contract.sdp.md:80 computes \"contribution minus the marker's recorded contribution\" and :81 reads only the new aggregate key, so the orderTotals example at projection-contract.sdp.md:84, with placed 100 moving to cancelled 100, leaves the old total and gives the new key zero. The issue and two-key repair remain right; derive marker identity from the full source identity, including contextId, and do not create a row for null. The build must show: Pure contribution tests and convex-test transitions must cover first contribution, same-key update, key change, null, stale DTO and live/backfill race; old and new values and counts must equal a rebuild after every transition, without creating a null-contribution row.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-02: At HEAD, projection-contract.sdp.md:80 computes \"contribution minus the marker's recorded contribution\" and :81 reads only the new aggregate key, so the orderTotals example at projection-contract.sdp.md:84, with placed 100 moving to cancelled 100, leaves the old total and gives the new key zero. The issue and two-key repair remain right; derive marker identity from the full source identity, including contextId, and do not create a row for null. The build must show: Pure contribution tests and convex-test transitions must cover first contribution, same-key update, key change, null, stale DTO and live/backfill race; old and new values and counts must equal a rebuild after every transition, without creating a null-contribution row. Close of round 3, 4c, on paper: the aggregate write splits on a key change, deletes a row whose count reaches 0 and creates none for a null contribution; built when a view needs the aggregate form (OD-075).", "slice": "S4", "detector": "convex-test" }, @@ -3113,8 +3114,8 @@ ], "issue": "`applyAggregate` takes `input: { tenantId: string; entityKey: string; dto: D; versions: StreamVersion[]; mode: ApplyMode; generations: number[] }` and reads the marker 'by `by_entity` with the tenant, read model, generation and `entityKey`'. No Spec says where `entityKey` comes from. `AggregateProjection⟨D⟩` has `aggregateKeyOf` and `contribute` and no entity key function; the pipeline's step 9 says only that it calls `applyAggregate` 'with the entry's DTO and version'; the rebuild's `step4` says 'for each item it calls `applyProjection` or `applyAggregate` in backfill mode'. The term `entityKey` appears in the corpus only in that signature and in the marker table. The marker exists so that 'live commands and backfill never count one entity twice', which holds only if the live caller and the backfill caller, pinned in two different Specs, derive the same key; a stream ID alone is also not unique when an aggregate is bound to two stream types.", "fix": "In `spec:application.projection-contract` add `entityKeyOf(tenantId: string, dto: D): string` to `AggregateProjection⟨D⟩` and have `applyAggregate` compute it, dropping `entityKey` from its input; or state once that `entityKey` is `contextId + \":\" + streamType + \":\" + streamId` of `dto.version`. Name the rule in the pipeline's step 9 and the rebuild's `step4`.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-02: At HEAD, projection-contract.sdp.md:78 requires \"entityKey: string\" but :70 gives AggregateProjection only aggregateKeyOf and contribute, so live and backfill callers cannot derive one declared identity. The issue holds; the fix's entityKeyOf alternative remains right, while its raw colon concatenation needs an injective encoding of the full source identity. The build must show: Compiled callers and convex-test must derive identical marker identities for live writes and backfill, with equal local stream IDs in distinct contexts and stream types remaining distinct.", + "status": "partially-fixed", + "note": "the platform review of 2026-10-02, APPL-02: At HEAD, projection-contract.sdp.md:78 requires \"entityKey: string\" but :70 gives AggregateProjection only aggregateKeyOf and contribute, so live and backfill callers cannot derive one declared identity. The issue holds; the fix's entityKeyOf alternative remains right, while its raw colon concatenation needs an injective encoding of the full source identity. The build must show: Compiled callers and convex-test must derive identical marker identities for live writes and backfill, with equal local stream IDs in distinct contexts and stream types remaining distinct. Close of round 3, 4c, on paper: entityKey is JSON.stringify([contextId, streamType, streamId]); built with the aggregate form.", "slice": "S4", "detector": "TypeScript compiler" }, @@ -3145,10 +3146,11 @@ ], "issue": "The batch size of a history view is computed two ways in one Spec. `fnStartGeneration` computes the row's `batchSize` as `min(args.batchSize ?? 100, floor(4 MiB / (2 * rowBudgetBytes)))` for every read model, which is 100 at the default 16 KiB row budget. `limitHistoryBatch` says `min(500, floor(4 MiB / (2 * rowBudgetBytes)))` events per history batch, '128 at the default 16 KiB row budget', and that 'the count is stored on the generation row as its `batchSize`'. The stored number for a history view at defaults is 100 by the first bullet and 128 by the second, and the cap is 100 or 500.", "fix": "In `spec:application.rebuild` make `fnStartGeneration` state both defaults: `args.batchSize ?? 100` entities for a `Projection` or `AggregateProjection`, `args.batchSize ?? 500` events for a `HistoryProjection`, each capped by `floor(4 MiB / (2 * rowBudgetBytes))`; or lower `limitHistoryBatch` to the same 100.", - "status": "open", - "note": "", + "status": "fixed", + "note": "Close of round 3, 4c: one batch-size formula, counted in events for a history view, 100 at 16 KiB and 32 at 64 KiB.", "slice": "S4", - "detector": "judgement" + "detector": "judgement", + "unreviewed": true }, { "round": 3, @@ -3532,8 +3534,8 @@ ], "issue": "Lead 2, read for this lens: three small mismatches around the newest form. First, `applyHistoryProjection` takes `input: { tenantId: string; events: EventEnvelope[]; mode: ApplyMode; generations: number[] }` and its description never reads `mode`; for `applyAggregate` the contract says '`mode` is accepted for symmetry with `applyProjection` and read by nothing', for the history helper it says nothing, while callers pass 'backfill mode' and the live modes. Second, the pipeline's step 9 calls it 'with the entry's `events` and version', and the helper has no version argument; it reads `streamVersion` off each envelope. Third, the adapter's `streams` entry carries every envelope the call wrote, 'baselines included', and the `history` query returns baseline events 'like any other', so `HistoryProjection.keyOf` receives events of the reserved type `baseline`, whose payload is a whole migrated state of up to 512 KiB, on both paths; the contract speaks only of domain events and does not say that a baseline is skipped or what it means to a view whose fold reads history.", "fix": "In `spec:application.projection-contract` state that `applyHistoryProjection` ignores `mode` (or drop the field), that it reads versions from the envelopes, and that it drops events of type `baseline` before `keyOf`, with one sentence on why a meaning migration of a source needs a new generation of the view. In `spec:command.command-pipeline` step 9 write 'with the entry's `events`'.", - "status": "open", - "note": "", + "status": "partially-fixed", + "note": "Close of round 3, 4c, on paper: mode is read by nothing, baseline events are dropped before keyOf, the history form keeps a single version with a delegated question; the step 9 wording lands after the audit lane's merge.", "slice": "S4", "detector": "judgement" }, From 12b6186bb4ac93cfcb926f9b5aaf2b30a22213f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 09:59:09 +0200 Subject: [PATCH 130/155] design: 4a the diagnostic sink is configuration Co-Authored-By: Claude Opus 5.5 --- design/specs/operations/baseline-operations.sdp.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/specs/operations/baseline-operations.sdp.md b/design/specs/operations/baseline-operations.sdp.md index 7b913c7..922c326 100644 --- a/design/specs/operations/baseline-operations.sdp.md +++ b/design/specs/operations/baseline-operations.sdp.md @@ -86,7 +86,7 @@ Optimistic-concurrency retries cannot be observed from inside a mutation, becaus - indexAuditByRequestKey: `.index("by_request_key", ["tenantId", "requestKey"])` (E-43) - indexAuditByRequestKeyUse: diagnosis by request key when the receipt has expired (D19, D6) - typeDiagnosticRecord: [extension] `type DiagnosticRecord = { tenantId: string; commandType: string; namespace: CallerNamespace; kind: "applied" | "businessFailure" | "rejection" | "transient" | "technical"; code?: string; replayed: boolean; operationId?: string; requestKey?: string; correlationId?: string; actorKind?: ActorKind; actorId?: string; causedBy?: CausedBy; eventsAppended: number; readModelRows: number; versions: StreamVersion[] }`, where `kind` is the outcome, written `technical` for a technical failure, or `transient` for a transient refusal, which is not an outcome, as `classifyThrown` of `spec:command.outcome-boundary` names those two, and `causedBy` is the envelope's `CausedBy` (E-43, D19, D4) -- typeDiagnosticSink: [extension] `type DiagnosticSink = (line: string) => void` and `const consoleSink: DiagnosticSink = (line) => console.log(line)`; the pipeline passes the declaration's `diagnosticSink`, or `consoleSink` when the declaration names none (E-43, D19) +- typeDiagnosticSink: [extension] `type DiagnosticSink = (line: string) => void` and `const consoleSink: DiagnosticSink = (line) => console.log(line)`; the pipeline passes the declaration's `diagnosticSink`, or `consoleSink` when the declaration names none; a composition names a sink to route or reshape its lines, so the field is configuration and not a callback that exists only for a test, and the fixture composition names one to break it (E-43, D19, E-15) - diagnosticOnce: [extension] every execution of a command entry's handler emits exactly one record: at step 11, at step 5 before a duplicate is returned, or in the entry's outermost catch; a call that the entry's argument validators refuse runs no handler and emits none, its completion record in the function log being its trace; a record is an observation of one execution, written before the mutation ends, so an execution refused at commit by an optimistic-concurrency conflict keeps its line in its own completion record, the OCC rerun writes its own, and the receipt stays the authority for what committed (E-43, D19, F1) - diagnosticSourcesReturned: [extension] for a command that returns at step 11, `tenantId`, `namespace`, `requestKey` and `correlationId` come from the `PipelineCall`, `actorKind` and `actorId` from its `actor.kind` and `actor.id`, `commandType` from the declaration's `name`, `kind` from the executor's outcome, `replayed` is false, `operationId` and `causedBy` come from the `OperationRef` step 7 minted, `eventsAppended` is the sum of `appended` over the executor's `streams` entries, `readModelRows` is the count `writeReadModels` returns at step 9, 0 when the declaration binds no read model, `versions` are the outcome's versions, and `code` is absent (E-43, D19, D2) - diagnosticSourcesDuplicate: [extension] for a duplicate returned at step 5, the call's fields come as for a command that returns at step 11, with `causedBy` present only when the call states one, `kind` is the receipt's `outcome`, `replayed` is true, `operationId` and `versions` come from the receipt, `eventsAppended` and `readModelRows` are 0 because the execution did no work, and `code` is absent (E-43, D6, D19) From 39e54720d8c12c8ec54b542733ff97f8e8406681 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:02:04 +0200 Subject: [PATCH 131/155] test: 4a the proof of audit and the diagnostic --- tests/native/baseline-operations-steps.ts | 226 +++++++ ...ine-operations.broken-audit-aborts.test.ts | 35 ++ ...rations.broken-metrics-never-abort.test.ts | 39 ++ tests/pure/diagnostic.test.ts | 220 +++++++ tests/simulator/diagnostic-context.ts | 54 ++ tests/simulator/diagnostic-support.ts | 140 +++++ tests/simulator/diagnostic.test.ts | 558 ++++++++++++++++++ 7 files changed, 1272 insertions(+) create mode 100644 tests/native/baseline-operations-steps.ts create mode 100644 tests/native/baseline-operations.broken-audit-aborts.test.ts create mode 100644 tests/native/baseline-operations.broken-metrics-never-abort.test.ts create mode 100644 tests/pure/diagnostic.test.ts create mode 100644 tests/simulator/diagnostic-context.ts create mode 100644 tests/simulator/diagnostic-support.ts create mode 100644 tests/simulator/diagnostic.test.ts diff --git a/tests/native/baseline-operations-steps.ts b/tests/native/baseline-operations-steps.ts new file mode 100644 index 0000000..7d5525a --- /dev/null +++ b/tests/native/baseline-operations-steps.ts @@ -0,0 +1,226 @@ +import type { FunctionReturnType } from "convex/server"; +import { ConvexError } from "convex/values"; +import { expect } from "vitest"; +import { api } from "../../fixture/convex/_generated/api.js"; +import { permissions } from "../../fixture/convex/depotCommands.js"; +import { + auditFaultOrderId, + sinkFaultTenant, + orderPermission, +} from "../../fixture/convex/orders.js"; +import type { CompletionRecord } from "../../harness/admin.js"; +import type { Backend } from "../../harness/backend.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import { fixtureBackend, required } from "../../harness/native.js"; +import { classifyThrown } from "../../src/command/index.js"; + +type Answer = FunctionReturnType; +export interface AuditWorld { + backend?: Backend; + client?: ReturnType; + tenantId?: string; + orderId?: string; + before?: Awaited>; + answer?: Answer; + error?: unknown; + completion?: CompletionRecord; +} +async function tables(backend: Backend) { + return { + events: await backend.admin.readTable("events", { component: "depot" }), + streams: await backend.admin.readTable("streams", { component: "depot" }), + receipts: await backend.admin.readTable("receipts"), + audits: await backend.admin.readTable("auditRecords"), + }; +} +export async function prepare(world: AuditWorld, audit: string) { + expect(audit).toBe("a mandatory audit record"); + const backend = await fixtureBackend(); + world.backend = backend; + expect(sinkFaultTenant).toBe("tenant-sink-fault"); + expect(auditFaultOrderId).toBe("order-audit-fault"); + const client = ordinaryClient(backend.url, { + token: await backend.issuer.token("user-1"), + }); + world.client = client; + for (const tenantId of ["tenant-all-1", sinkFaultTenant]) { + for (const permission of [orderPermission, permissions.stock]) + await backend.admin.run("grants:grant", { + tenantId, + principalKind: "human", + principalId: `${backend.issuer.issuer}|user-1`, + permission, + grantedBy: "operator", + }); + await client.mutation(api.depotCommands.addStock, { + tenantId, + input: { lines: [{ productId: "p-1", quantity: 10 }] }, + }); + } +} +export function select(world: AuditWorld, subsystem: string) { + expect(["mandatory audit", "metrics and logging"]).toContain(subsystem); + world.tenantId = + subsystem === "mandatory audit" ? "tenant-all-1" : sinkFaultTenant; + world.orderId = + subsystem === "mandatory audit" ? auditFaultOrderId : "order-all-1"; +} +async function send( + world: AuditWorld, + tenantId: string, + orderId: string, + requestKey: string, +) { + const backend = required(world.backend, "backend"); + const mark = await backend.admin.logMark(); + let answer: Answer | undefined; + let error: unknown; + try { + answer = await required(world.client, "client").mutation( + api.orders.placeOrder, + { + tenantId, + requestKey, + input: { + orderId, + title: "Order", + lines: [{ productId: "p-1", quantity: 1 }], + }, + }, + ); + } catch (thrown) { + error = thrown; + } + const own = (record: CompletionRecord) => + record.identifier === "orders:placeOrder" && record.componentPath === null; + const records = await backend.admin.completionsSince(mark, (entries) => + entries.some(own), + ); + return { + answer, + error, + completion: required(records.find(own), "PlaceOrder completion"), + }; +} +export async function run(world: AuditWorld) { + world.before = await tables(required(world.backend, "backend")); + Object.assign( + world, + await send( + world, + required(world.tenantId, "tenant"), + required(world.orderId, "order"), + "all-1-key", + ), + ); +} +export async function assertResult(world: AuditWorld, result: string) { + const after = await tables(required(world.backend, "backend")); + if (result === "rolls back") { + expect(world.answer).toBeUndefined(); + expect(after).toEqual(required(world.before, "tables before command")); + return; + } + expect(result).toBe("commits"); + expect(world.error).toBeUndefined(); + expect(world.answer).toMatchObject({ kind: "applied", replayed: false }); + const { operationId } = world.answer as { operationId: string }; + const events = after.events.filter( + (event) => event.operationId === operationId, + ); + expect(events).toHaveLength(2); + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + streamType: "document", + streamId: "order-all-1", + }), + expect.objectContaining({ streamType: "stock", streamId: "p-1" }), + ]), + ); + expect(after.events).toHaveLength( + required(world.before, "tables before command").events.length + 2, + ); + expect(after.receipts).toEqual([ + expect.objectContaining({ + requestKey: "all-1-key", + operationId, + outcome: "applied", + }), + ]); + expect(after.audits).toEqual([ + expect.objectContaining({ + tenantId: "tenant-sink-fault", + operationId, + requestKey: "all-1-key", + commandType: "PlaceOrder", + kind: "business", + decision: "applied", + subject: { + contextId: "depot", + streamType: "document", + streamId: "order-all-1", + }, + }), + ]); +} +export async function assertFailure(world: AuditWorld, surfaced: string) { + const record = required(world.completion, "completion"); + const diagnostics = record.logLines.filter((line) => + line.includes("diagnostic {"), + ); + const gaps = record.logLines.filter((line) => + line.includes("diagnostic gap"), + ); + if (surfaced === "returned to the caller as a technical failure") { + expect(world.error).toBeInstanceOf(Error); + expect(world.error).not.toBeInstanceOf(ConvexError); + expect(classifyThrown(world.error).kind).toBe("technical"); + expect(record.error).toContain("auditRecords"); + expect(record.error).toContain("auditFault"); + expect(diagnostics).toHaveLength(1); + for (const text of ["technical", "tenant-all-1", "all-1-key"]) + expect(diagnostics[0]).toContain(text); + expect(diagnostics[0]).not.toContain("operationId"); + expect(gaps).toEqual([]); + } else { + expect(surfaced).toBe( + "reported as a diagnostic gap without touching the write", + ); + expect(record.error).toBeNull(); + expect(gaps).toHaveLength(1); + expect(gaps[0]).toContain( + (world.answer as { operationId: string }).operationId, + ); + expect(gaps[0]).toContain("applied"); + expect(diagnostics).toEqual([]); + } + const control = await send( + world, + "tenant-all-1", + "order-control", + "all-1-control", + ); + expect(control.error).toBeUndefined(); + expect(control.answer).toMatchObject({ kind: "applied", replayed: false }); + const operationId = (control.answer as { operationId: string }).operationId; + const audits = (await tables(required(world.backend, "backend"))).audits; + expect(audits).toHaveLength( + surfaced === "returned to the caller as a technical failure" ? 1 : 2, + ); + expect(audits.filter((row) => row.operationId === operationId)).toEqual([ + expect.objectContaining({ operationId, decision: "applied" }), + ]); + expect(control.completion.error).toBeNull(); + const lines = control.completion.logLines.filter((line) => + line.includes("diagnostic {"), + ); + expect(lines).toHaveLength(1); + expect(lines[0]).toContain(operationId); + expect(lines[0]).toContain("applied"); + expect( + control.completion.logLines.filter((line) => + line.includes("diagnostic gap"), + ), + ).toEqual([]); +} diff --git a/tests/native/baseline-operations.broken-audit-aborts.test.ts b/tests/native/baseline-operations.broken-audit-aborts.test.ts new file mode 100644 index 0000000..e57c67d --- /dev/null +++ b/tests/native/baseline-operations.broken-audit-aborts.test.ts @@ -0,0 +1,35 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { brokenAuditAbortsContract as contract } from "../../generated/contracts/operations.baseline-operations.broken-audit-aborts.contract.js"; +import { + prepare, + select, + run, + assertResult, + assertFailure, + type AuditWorld, +} from "./baseline-operations-steps.js"; +const anchor = specTest({ + id: testAnchorId("test:operations.baseline-operations.broken-audit-aborts"), + verifies: ref("spec:operations.baseline-operations.broken-audit-aborts"), +}); +void anchor; +bindExample(contract, (): AuditWorld => ({}), { + "a valid command whose use case emits diagnostics and writes {audit}": async ( + world, + { audit }, + ) => prepare(world, audit), + "the {subsystem} subsystem is broken by fault injection": ( + world, + { subsystem }, + ) => select(world, subsystem), + "the command runs": async (world) => run(world), + "the command {result}": async (world, { result }) => + assertResult(world, result), + "the failure is {surfaced}": async (world, { surfaced }) => + assertFailure(world, surfaced), +}); diff --git a/tests/native/baseline-operations.broken-metrics-never-abort.test.ts b/tests/native/baseline-operations.broken-metrics-never-abort.test.ts new file mode 100644 index 0000000..50e57c5 --- /dev/null +++ b/tests/native/baseline-operations.broken-metrics-never-abort.test.ts @@ -0,0 +1,39 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { brokenMetricsNeverAbortContract as contract } from "../../generated/contracts/operations.baseline-operations.broken-metrics-never-abort.contract.js"; +import { + prepare, + select, + run, + assertResult, + assertFailure, + type AuditWorld, +} from "./baseline-operations-steps.js"; +const anchor = specTest({ + id: testAnchorId( + "test:operations.baseline-operations.broken-metrics-never-abort", + ), + verifies: ref( + "spec:operations.baseline-operations.broken-metrics-never-abort", + ), +}); +void anchor; +bindExample(contract, (): AuditWorld => ({}), { + "a valid command whose use case emits diagnostics and writes {audit}": async ( + world, + { audit }, + ) => prepare(world, audit), + "the {subsystem} subsystem is broken by fault injection": ( + world, + { subsystem }, + ) => select(world, subsystem), + "the command runs": async (world) => run(world), + "the command {result}": async (world, { result }) => + assertResult(world, result), + "the failure is {surfaced}": async (world, { surfaced }) => + assertFailure(world, surfaced), +}); diff --git a/tests/pure/diagnostic.test.ts b/tests/pure/diagnostic.test.ts new file mode 100644 index 0000000..d6e0e90 --- /dev/null +++ b/tests/pure/diagnostic.test.ts @@ -0,0 +1,220 @@ +import { afterEach, expect, test, vi } from "vitest"; +import { + consoleSink, + diagnosticLine, + diagnosticGapLine, + emitDiagnostic, + limitDiagnosticBytes, + type DiagnosticRecord, + type DiagnosticSink, +} from "../../src/operations/diagnostic.js"; +import * as operations from "../../src/operations/index.js"; +import { + fixtureDiagnosticSink, + sinkFaultTenant, + auditFaultOrderId, +} from "../../fixture/convex/orders.js"; + +afterEach(() => vi.restoreAllMocks()); +const record: DiagnosticRecord = { + tenantId: "tenant-ö", + commandType: "Command", + namespace: "worker", + kind: "applied", + replayed: false, + operationId: "operation", + requestKey: "request", + correlationId: "correlation", + actorKind: "service", + actorId: "actor", + causedBy: { kind: "command", commandType: "Cause" }, + eventsAppended: 2, + readModelRows: 1, + versions: [ + { + tenantId: "tenant-ö", + contextId: "context", + streamType: "stream", + streamId: "subject", + version: 102, + }, + ], +}; +const bytes = (line: string) => new TextEncoder().encode(line).length; +// An independent byte oracle. Array.from iterates code points, including supplementary characters. +function prefix(line: string) { + let result = ""; + for (const point of line) { + if (bytes(result + point) > 4096) break; + result += point; + } + return result; +} +test("pure: the diagnostic module exports the pinned API and 4096 byte bound", () => { + expect(limitDiagnosticBytes).toBe(4096); + expect(operations.consoleSink).toBe(consoleSink); + expect(operations.diagnosticLine).toBe(diagnosticLine); + expect(operations.diagnosticGapLine).toBe(diagnosticGapLine); + expect(operations.emitDiagnostic).toBe(emitDiagnostic); + expect(operations.limitDiagnosticBytes).toBe(4096); + const sink: operations.DiagnosticSink = consoleSink; + const typed: operations.DiagnosticRecord = record; + expect(sink).toBe(consoleSink); + expect(typed).toBe(record); +}); +test("pure: diagnosticLine orders only declared keys, omits absent fields and never adds duration", () => { + const full: DiagnosticRecord = { + ...record, + kind: "rejection", + code: "forbidden", + }; + const shuffled = Object.fromEntries( + Object.entries(full).reverse(), + ) as DiagnosticRecord; + Object.assign(shuffled, { duration: 123, durationMs: 456, extra: "secret" }); + const ordered = { + tenantId: record.tenantId, + commandType: record.commandType, + namespace: record.namespace, + kind: "rejection", + code: "forbidden", + replayed: false, + operationId: record.operationId, + requestKey: record.requestKey, + correlationId: record.correlationId, + actorKind: record.actorKind, + actorId: record.actorId, + causedBy: record.causedBy, + eventsAppended: 2, + readModelRows: 1, + versions: record.versions, + }; + expect(diagnosticLine(shuffled)).toBe( + `diagnostic ${JSON.stringify(ordered)}`, + ); + const minimal: DiagnosticRecord = { + tenantId: "t", + commandType: "C", + namespace: "public", + kind: "technical", + replayed: false, + eventsAppended: 0, + readModelRows: 0, + versions: [], + }; + expect(diagnosticLine(minimal)).toBe(`diagnostic ${JSON.stringify(minimal)}`); +}); +test("pure: a line at 4096 bytes is retained and one byte above cuts versions with omission overhead", () => { + const base = { ...record, versions: [] }; + const padding = 4096 - bytes(`diagnostic ${JSON.stringify(base)}`); + const exact = { ...base, tenantId: base.tenantId + "a".repeat(padding) }; + expect(bytes(diagnosticLine(exact))).toBe(4096); + expect(diagnosticLine(exact)).toBe(`diagnostic ${JSON.stringify(exact)}`); + const larger = { ...exact, tenantId: exact.tenantId + "a" }; + const { versions, ...fields } = larger; + expect(diagnosticLine(larger)).toBe( + prefix( + `diagnostic ${JSON.stringify({ ...fields, versionsOmitted: 0, versions })}`, + ), + ); +}); +test("pure: versions are cut to the longest fitting prefix with versionsOmitted immediately before versions", () => { + const versions = Array.from({ length: 110 }, (_, i) => ({ + ...record.versions[0]!, + streamId: `subject-${i}-🦊`, + version: 100 + i, + })); + const input = { ...record, versions }; + const { versions: all, ...fields } = input; + const candidate = (kept: number) => + `diagnostic ${JSON.stringify({ ...fields, versionsOmitted: all.length - kept, versions: all.slice(0, kept) })}`; + const fitting = Array.from( + { length: all.length + 1 }, + (_, kept) => kept, + ).filter((kept) => bytes(candidate(kept)) <= 4096); + const kept = fitting.at(-1)!; + expect(kept).toBeGreaterThan(0); + expect(kept).toBeLessThan(all.length); + expect(diagnosticLine(input)).toBe(candidate(kept)); + expect(bytes(candidate(kept + 1))).toBeGreaterThan(4096); +}); +test("pure: an oversized record is never dropped and the final cut preserves whole code points", () => { + const input = { ...record, tenantId: "a🦊é".repeat(1600) }; + const { versions, ...fields } = input; + const whole = `diagnostic ${JSON.stringify({ ...fields, versionsOmitted: versions.length, versions: [] })}`; + const line = diagnosticLine(input); + expect(line).toBe(prefix(whole)); + expect(bytes(line)).toBeLessThanOrEqual(4096); + expect( + bytes(line + Array.from(whole.slice(line.length))[0]!), + ).toBeGreaterThan(4096); + expect(line).not.toContain("�"); +}); +test("pure: diagnosticGapLine includes only its ordered identifiers with optional operationId and the same byte cut", () => { + expect(diagnosticGapLine(record)).toBe( + 'diagnostic gap {"tenantId":"tenant-ö","commandType":"Command","kind":"applied","operationId":"operation"}', + ); + const without = { ...record }; + delete without.operationId; + expect(diagnosticGapLine(without)).toBe( + 'diagnostic gap {"tenantId":"tenant-ö","commandType":"Command","kind":"applied"}', + ); + const huge = { ...record, tenantId: "🦊é".repeat(2000) }; + const whole = `diagnostic gap ${JSON.stringify({ tenantId: huge.tenantId, commandType: huge.commandType, kind: huge.kind, operationId: huge.operationId })}`; + expect(diagnosticGapLine(huge)).toBe(prefix(whole)); +}); +test("pure: a working sink receives one line, with no second log or gap; the default is consoleSink", () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const sink: DiagnosticSink = vi.fn(); + expect(emitDiagnostic(record, sink)).toBeUndefined(); + expect(sink).toHaveBeenCalledExactlyOnceWith(diagnosticLine(record)); + expect(log).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + expect(emitDiagnostic(record)).toBeUndefined(); + expect(log).toHaveBeenCalledExactlyOnceWith(diagnosticLine(record)); + expect(error).not.toHaveBeenCalled(); +}); +for (const thrown of [ + new Error("sink broke"), + "string", + null, + undefined, + { broken: true }, +]) { + test(`pure: emitDiagnostic swallows ${String(thrown)} and writes one gap, even if console.error throws`, () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const sink = vi.fn(() => { + throw thrown; + }); + expect(emitDiagnostic(record, sink)).toBeUndefined(); + expect(sink).toHaveBeenCalledExactlyOnceWith(diagnosticLine(record)); + expect(error).toHaveBeenCalledExactlyOnceWith(diagnosticGapLine(record)); + expect(log).not.toHaveBeenCalled(); + error.mockClear().mockImplementation(() => { + throw new Error("gap sink broke"); + }); + sink.mockClear(); + expect(emitDiagnostic(record, sink)).toBeUndefined(); + expect(sink).toHaveBeenCalledExactlyOnceWith(diagnosticLine(record)); + expect(error).toHaveBeenCalledExactlyOnceWith(diagnosticGapLine(record)); + expect(log).not.toHaveBeenCalled(); + }); +} +test("pure: the fixture sink faults only for the pinned tenant and otherwise uses consoleSink", () => { + expect(auditFaultOrderId).toBe("order-audit-fault"); + expect(sinkFaultTenant).toBe("tenant-sink-fault"); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const bad = diagnosticLine({ ...record, tenantId: sinkFaultTenant }); + expect(() => fixtureDiagnosticSink(bad)).toThrow( + /^Fault injected: the diagnostic sink is broken$/, + ); + expect(log).not.toHaveBeenCalled(); + const good = diagnosticLine({ + ...record, + tenantId: `${sinkFaultTenant}-other`, + }); + fixtureDiagnosticSink(good); + expect(log).toHaveBeenCalledExactlyOnceWith(good); +}); diff --git a/tests/simulator/diagnostic-context.ts b/tests/simulator/diagnostic-context.ts new file mode 100644 index 0000000..5c802e7 --- /dev/null +++ b/tests/simulator/diagnostic-context.ts @@ -0,0 +1,54 @@ +import { v } from "convex/values"; +import { internalQueryGeneric } from "convex/server"; +import { + defineOperation, + planned, + type StreamRegistration, +} from "../../src/context/index.js"; +import type { DomainEvent } from "../../src/kernel/index.js"; +import { journal } from "../../fixture/convex/depot/streams.js"; + +// A context registered only by these tests. Each amount appends one event on the same stream. +const counter: StreamRegistration< + number, + number[], + DomainEvent<"added", { amount: number }>, + null +> = { + decider: { + streamType: "counter", + initial: () => 0, + decide: (_state, amounts) => ({ + kind: "applied", + result: null, + events: amounts.map((amount) => ({ + eventType: "added", + eventSchemaVersion: 1, + payload: { amount }, + })), + }), + evolve: (state, event) => state + event.payload.amount, + invariants: [], + }, + mapping: { kind: "single", budgetBytes: 4096, isDeleted: () => false }, + stateSchemaVersion: 1, + eventValidators: { added: v.object({ amount: v.number() }) }, + dto: v.number(), + toDto: (state) => state, +}; +export const add = defineOperation<{ amounts: number[] }, null>(journal, { + name: "add", + streams: [counter], + input: { amounts: v.array(v.number()) }, + returns: v.null(), + plan: ({ amounts }) => [planned(counter, "counter", amounts)], + combine: () => null, + maxStreams: 1, +}); +export const stored = internalQueryGeneric({ + args: {}, + handler: async (ctx) => ({ + streams: await ctx.db.query("streams").collect(), + events: await ctx.db.query("events").collect(), + }), +}); diff --git a/tests/simulator/diagnostic-support.ts b/tests/simulator/diagnostic-support.ts new file mode 100644 index 0000000..907a696 --- /dev/null +++ b/tests/simulator/diagnostic-support.ts @@ -0,0 +1,140 @@ +import { convexTest } from "convex-test"; +import { makeFunctionReference } from "convex/server"; +import { v } from "convex/values"; +import { vi } from "vitest"; +import schema from "../../fixture/convex/schema.js"; +import depotSchema from "../../fixture/convex/depot/schema.js"; +import { components } from "../../fixture/convex/_generated/api.js"; +import { + publicCommand, + internalCommand, + insertGrant, + type Actor, + type CommandDeclaration, + type InternalCommandArgs, + type PublicCommandArgs, + type CommandResponse, +} from "../../src/command/index.js"; +import type { OperationRef, StreamDto } from "../../src/context/index.js"; +import type { DiagnosticRecord } from "../../src/operations/index.js"; + +export const actor = { + kind: "service", + id: "service-actor", + issuer: "service-issuer", +} as const; +export const cause = { + kind: "event", + tenantId: "cause-tenant", + contextId: "cause-context", + eventId: "cause-event", +} as const; +export const input = { documentId: "document", title: "Title" }; +export const call: InternalCommandArgs = { + tenantId: "call-tenant", + namespace: "worker", + actor, + requestKey: "call-key", + correlationId: "call-correlation", + causedBy: cause, + input, +}; +export const issuer = "https://diagnostic.test"; +export const human = { kind: "human", id: `${issuer}|human` } as const; +export const internalRef = makeFunctionReference< + "mutation", + InternalCommandArgs, + CommandResponse +>("diagnostic:internal"); +export const publicRef = makeFunctionReference< + "mutation", + PublicCommandArgs, + CommandResponse +>("diagnostic:public"); +export function entry(appended = 1, streamId = "document"): StreamDto { + const version = { + tenantId: call.tenantId, + contextId: "depot", + streamType: "document", + streamId, + version: 100 + appended, + }; + return { + version, + appended, + created: false, + events: [], + dto: { + documentId: streamId, + title: "Title", + status: "draft", + amendments: 100, + version, + }, + }; +} +export function diagnosticApp( + change: Partial> = {}, +) { + const lines: string[] = []; + const operations: OperationRef[] = []; + const decl: CommandDeclaration = { + name: "DiagnosticCommand", + contractVersion: 1, + input: v.object({ documentId: v.string(), title: v.string() }), + output: v.null(), + permission: { permission: "documents" }, + writes: [{ contextId: "depot", streamType: "document" }], + audit: { kind: "security" }, + rejections: ["titleRequired"], + diagnosticSink: (line: string) => { + lines.push(line); + }, + executor: async (ctx, { tenantId, actor, operation, input }) => { + operations.push(operation); + const result = await ctx.runMutation( + components.depot.operations.createDocuments, + { tenantId, actor, operation, input: { documents: [input] } }, + ); + return { ...result, result: null }; + }, + ...change, + }; + const execute = vi.fn(decl.executor); + decl.executor = execute; + const t = convexTest(schema, { + ...import.meta.glob("../../fixture/convex/**/*.ts"), + "../../fixture/convex/diagnostic.ts": async () => ({ + public: publicCommand(decl), + internal: internalCommand(decl), + }), + }); + t.registerComponent("depot", depotSchema, { + ...import.meta.glob("../../fixture/convex/depot/**/*.ts"), + "../../fixture/convex/depot/diagnosticContext.ts": () => + import("./diagnostic-context.js"), + }); + const grant = async (principal: Actor = actor) => + t.run((ctx) => + insertGrant(ctx, { + tenantId: call.tenantId, + principalKind: principal.kind, + principalId: principal.id, + permission: "documents", + grantedBy: "operator", + }), + ); + const stored = () => + t.run(async (ctx) => ({ + receipts: await ctx.db.query("receipts").collect(), + audits: await ctx.db.query("auditRecords").collect(), + rows: await ctx.db.query("documentSummaries").collect(), + })); + const records = (): DiagnosticRecord[] => + lines.map((line) => { + if (!line.startsWith("diagnostic ")) + throw new Error(`Unexpected diagnostic: ${line}`); + return JSON.parse(line.slice("diagnostic ".length)) as DiagnosticRecord; + }); + return { t, grant, stored, records, lines, operations, decl, execute }; +} diff --git a/tests/simulator/diagnostic.test.ts b/tests/simulator/diagnostic.test.ts new file mode 100644 index 0000000..a966f97 --- /dev/null +++ b/tests/simulator/diagnostic.test.ts @@ -0,0 +1,558 @@ +import { componentsGeneric, type FunctionReference } from "convex/server"; +import type { + OperationRef, + OperationOutcome, +} from "../../src/context/index.js"; +import { ConvexError } from "convex/values"; +import { afterEach, beforeEach, expect, test, vi } from "vitest"; +import { components } from "../../fixture/convex/_generated/api.js"; +import { documentSummary } from "../../fixture/convex/summaries.js"; +import { classifyThrown, runPipeline } from "../../src/command/index.js"; +import { + diagnosticApp, + actor, + cause, + call, + human, + issuer, + internalRef, + publicRef, + entry, +} from "./diagnostic-support.js"; +import { caught } from "./gate-support.js"; + +beforeEach(() => vi.stubEnv("MAINTENANCE_MODE", undefined)); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); +const fields = { + tenantId: call.tenantId, + namespace: call.namespace, + commandType: "DiagnosticCommand", + requestKey: call.requestKey, + correlationId: call.correlationId, + actorKind: actor.kind, + actorId: actor.id, + causedBy: cause, +}; +const noWork = { + replayed: false, + eventsAppended: 0, + readModelRows: 0, + versions: [], +}; +for (const kind of ["applied", "businessFailure"] as const) { + test(`convex-test: ${kind} emits once from the call, operation and outcome, audits once, and duplicate uses the new call and old receipt`, async () => { + let refuseNew = false; + const a = diagnosticApp({ + admission: async () => + refuseNew ? { admitted: false, code: "capacity" } : { admitted: true }, + }); + const execute = a.decl.executor; + a.decl.executor = async (ctx, args) => ({ + ...(await execute(ctx, args)), + kind, + }); + await a.grant(); + const errorLog = vi.spyOn(console, "error").mockImplementation(() => {}); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const first = await a.t.mutation(internalRef, call); + expect(a.operations).toHaveLength(1); + expect.soft(a.records()).toEqual([ + { + ...fields, + kind, + replayed: false, + operationId: first.operationId, + eventsAppended: 1, + readModelRows: 0, + versions: first.versions, + }, + ]); + expect(a.operations[0]).toEqual({ + operationId: first.operationId, + causedBy: cause, + correlationId: call.correlationId, + }); + const before = await a.stored(); + expect(before.audits).toEqual([ + expect.objectContaining({ + tenantId: call.tenantId, + actor, + operationId: first.operationId, + requestKey: call.requestKey, + commandType: "DiagnosticCommand", + kind: "security", + decision: kind, + causedBy: cause, + subject: { + contextId: "depot", + streamType: "document", + streamId: "document", + }, + }), + ]); + const secondActor = { + kind: "agent", + id: "second-actor", + issuer: "second-issuer", + } as const; + await a.grant(secondActor); + const secondCause = { + kind: "command", + commandType: "DifferentCause", + } as const; + a.lines.length = 0; + refuseNew = true; + const duplicate = await a.t.mutation(internalRef, { + ...call, + actor: secondActor, + correlationId: "second-correlation", + causedBy: secondCause, + }); + expect(duplicate).toMatchObject({ + kind, + replayed: true, + operationId: first.operationId, + versions: first.versions, + result: null, + }); + expect.soft(a.records()).toEqual([ + { + ...fields, + actorKind: secondActor.kind, + actorId: secondActor.id, + correlationId: "second-correlation", + causedBy: secondCause, + kind, + replayed: true, + operationId: first.operationId, + eventsAppended: 0, + readModelRows: 0, + versions: first.versions, + }, + ]); + expect(a.execute).toHaveBeenCalledTimes(1); + expect(await a.stored()).toEqual(before); + a.lines.length = 0; + const without = { ...call }; + delete without.causedBy; + delete without.correlationId; + await a.t.mutation(internalRef, without); + const plainFields = { + tenantId: call.tenantId, + namespace: call.namespace, + commandType: "DiagnosticCommand", + requestKey: call.requestKey, + actorKind: actor.kind, + actorId: actor.id, + }; + expect.soft(a.records()).toEqual([ + { + ...plainFields, + kind, + replayed: true, + operationId: first.operationId, + eventsAppended: 0, + readModelRows: 0, + versions: first.versions, + }, + ]); + expect(await a.stored()).toEqual(before); + expect(errorLog).not.toHaveBeenCalled(); + expect(log).not.toHaveBeenCalled(); + }); +} +test("convex-test: public success defaults the namespace, actor and command cause, omits absent call fields, and uses consoleSink", async () => { + const a = diagnosticApp(); + delete a.decl.diagnosticSink; + await a.grant(human); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const response = await a.t + .withIdentity({ issuer, subject: "human" }) + .mutation(publicRef, { tenantId: call.tenantId, input: call.input }); + expect(log).toHaveBeenCalledTimes(1); + expect(log.mock.calls[0]).toHaveLength(1); + expect( + JSON.parse(String(log.mock.calls[0]![0]).slice("diagnostic ".length)), + ).toEqual({ + tenantId: call.tenantId, + namespace: "public", + commandType: "DiagnosticCommand", + kind: "applied", + replayed: false, + operationId: response.operationId, + actorKind: "human", + actorId: human.id, + causedBy: { kind: "command", commandType: "DiagnosticCommand" }, + eventsAppended: 1, + readModelRows: 0, + versions: response.versions, + }); + expect((await a.stored()).audits).toHaveLength(1); + expect(error).not.toHaveBeenCalled(); +}); +for (const ending of [ + "rejection", + "transient", + "technical", + "undeclared rejection", +] as const) { + test(`convex-test: internal ${ending} emits one normalized failure with call fields, no operation or audit`, async () => { + const a = diagnosticApp(); + await a.grant(); + const error = new Error("executor broke"); + if (ending === "transient") + a.decl.admission = async () => ({ + admitted: false, + code: "rateLimited", + retryAfterMs: 17, + }); + else + a.decl.executor = async (ctx, args) => { + // A failure after real context writes must still report zero committed work. + await ctx.runMutation(components.depot.operations.createDocuments, { + tenantId: args.tenantId, + actor: args.actor, + operation: args.operation, + input: { documents: [args.input] }, + }); + if (ending === "technical") throw error; + throw new ConvexError({ + code: ending === "rejection" ? "titleRequired" : "unknownCode", + message: "refused", + }); + }; + const gap = vi.spyOn(console, "error").mockImplementation(() => {}); + const thrown = await caught(a.t.mutation(internalRef, call)); + const kind = ending === "undeclared rejection" ? "technical" : ending; + expect(classifyThrown(thrown).kind).toBe(kind); + if (ending === "technical") expect(thrown).toBe(error); + if (kind === "technical") expect(thrown).not.toBeInstanceOf(ConvexError); + else + expect(thrown).toMatchObject({ + data: { + kind, + code: kind === "rejection" ? "titleRequired" : "rateLimited", + }, + }); + expect.soft(a.records()).toEqual([ + { + ...fields, + ...noWork, + kind, + ...(kind === "technical" + ? {} + : { code: kind === "rejection" ? "titleRequired" : "rateLimited" }), + }, + ]); + expect(await a.stored()).toEqual({ receipts: [], audits: [], rows: [] }); + expect( + await a.t.run((ctx) => + ctx.runQuery(components.depot.queries.document.get, { + tenantId: call.tenantId, + streamId: "document", + }), + ), + ).toBeNull(); + expect(gap).not.toHaveBeenCalled(); + }); +} +for (const authenticated of [false, true]) { + test(`convex-test: public refusal ${authenticated ? "after" : "before"} authentication includes only the established actor`, async () => { + const a = diagnosticApp(); + const t = authenticated + ? a.t.withIdentity({ issuer, subject: "human" }) + : a.t; + const thrown = await caught( + t.mutation(publicRef, { + tenantId: call.tenantId, + requestKey: "public-key", + correlationId: "public-correlation", + input: call.input, + }), + ); + const code = authenticated ? "forbidden" : "unauthenticated"; + expect(thrown).toMatchObject({ data: { kind: "rejection", code } }); + expect.soft(a.records()).toEqual([ + { + tenantId: call.tenantId, + commandType: "DiagnosticCommand", + namespace: "public", + kind: "rejection", + code, + requestKey: "public-key", + correlationId: "public-correlation", + ...noWork, + ...(authenticated ? { actorKind: "human", actorId: human.id } : {}), + }, + ]); + expect(a.execute).not.toHaveBeenCalled(); + expect(await a.stored()).toEqual({ receipts: [], audits: [], rows: [] }); + }); +} +for (const ref of [publicRef, internalRef]) { + test(`convex-test: validators refuse ${ref === publicRef ? "public" : "internal"} arguments without a diagnostic`, async () => { + const a = diagnosticApp(); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const gap = vi.spyOn(console, "error").mockImplementation(() => {}); + await expect( + a.t.mutation(ref, { ...call, tenantId: 5 } as never), + ).rejects.toThrow(); + expect.soft(a.records()).toEqual([]); + expect(a.execute).not.toHaveBeenCalled(); + expect(log).not.toHaveBeenCalled(); + expect(gap).not.toHaveBeenCalled(); + expect(await a.stored()).toEqual({ receipts: [], audits: [], rows: [] }); + }); +} +for (const failGap of [false, true]) { + test(`convex-test: a broken sink with ${failGap ? "broken" : "working"} gap logging preserves success and failure`, async () => { + const sink = vi.fn(() => { + throw null; + }); + let refuseNew = false; + const a = diagnosticApp({ + diagnosticSink: sink, + admission: async () => + refuseNew ? { admitted: false, code: "capacity" } : { admitted: true }, + }); + await a.grant(); + const gap = vi.spyOn(console, "error").mockImplementation(() => { + if (failGap) throw new Error("gap broke"); + }); + const result = await a.t.mutation(internalRef, call); + expect(result).toMatchObject({ kind: "applied", replayed: false }); + expect(sink).toHaveBeenCalledTimes(1); + expect(gap).toHaveBeenCalledExactlyOnceWith( + `diagnostic gap ${JSON.stringify({ tenantId: call.tenantId, commandType: "DiagnosticCommand", kind: "applied", operationId: result.operationId })}`, + ); + const before = await a.stored(); + expect(before.audits).toHaveLength(1); + expect(before.receipts).toHaveLength(1); + sink.mockClear(); + gap.mockClear(); + refuseNew = true; + await expect( + a.t.mutation(internalRef, { ...call, requestKey: "refused-key" }), + ).rejects.toMatchObject({ data: { kind: "transient", code: "capacity" } }); + expect(sink).toHaveBeenCalledTimes(1); + expect(gap).toHaveBeenCalledExactlyOnceWith( + `diagnostic gap ${JSON.stringify({ tenantId: call.tenantId, commandType: "DiagnosticCommand", kind: "transient" })}`, + ); + expect(await a.stored()).toEqual(before); + }); +} +for (const appended of [1, 2]) { + for (const rows of [0, 1, 2]) { + test(`convex-test: ${appended} appended at version 100 and ${rows} generation rows count actual work`, async () => { + const stream = entry(appended); + const a = diagnosticApp({ + executor: async () => ({ + kind: "applied", + result: null, + versions: [stream.version], + streams: [stream], + }), + ...(rows === 0 + ? {} + : { + readModels: [ + { + readModel: documentSummary, + source: { contextId: "depot", streamType: "document" }, + }, + ], + }), + }); + await a.grant(); + if (rows > 0) + await a.t.run(async (ctx) => { + for (let generation = 1; generation <= rows; generation++) { + await ctx.db.insert("generations", { + readModel: "documentSummary", + generation, + projectionVersion: 1, + state: generation === 1 ? "active" : "building", + pauseRequired: false, + fence: 0, + startedAt: 0, + startedBy: "operator", + changedAt: 0, + changedBy: "operator", + }); + // A live update writes an existing building row as well as the active row. + await ctx.db.insert("documentSummaries", { + tenantId: call.tenantId, + generation, + key: "document", + projectionVersion: 1, + sourceVersions: [{ ...stream.version, version: 100 }], + documentId: "document", + status: "draft", + title: "Before", + }); + } + }); + const response = await a.t.mutation(internalRef, call); + expect.soft(a.records()).toEqual([ + { + ...fields, + kind: "applied", + replayed: false, + operationId: response.operationId, + eventsAppended: appended, + readModelRows: rows, + versions: [stream.version], + }, + ]); + const stored = await a.stored(); + expect(stored.rows).toHaveLength(rows); + for (const row of stored.rows) + expect(row).toMatchObject({ + title: "Title", + sourceVersions: [stream.version], + }); + }); + } +} +test("convex-test: eventsAppended sums all streams rather than counting streams or final versions", async () => { + const streams = [entry(1, "first"), entry(2, "second")]; + const a = diagnosticApp({ + executor: async () => ({ + kind: "businessFailure", + result: null, + streams, + versions: streams.map((stream) => stream.version), + }), + }); + await a.grant(); + const response = await a.t.mutation(internalRef, call); + expect.soft(a.records()).toEqual([ + { + ...fields, + kind: "businessFailure", + replayed: false, + operationId: response.operationId, + eventsAppended: 3, + readModelRows: 0, + versions: response.versions, + }, + ]); + expect((await a.stored()).audits).toHaveLength(1); +}); + +for (const appended of [1, 2]) { + test(`convex-test: the journal appends ${appended} events to a stream already at version 100`, async () => { + const context = componentsGeneric().depot!.diagnosticContext!; + const add = context.add as FunctionReference< + "mutation", + "public", + { + tenantId: string; + actor: typeof actor; + operation: OperationRef; + input: { amounts: number[] }; + }, + OperationOutcome + >; + const stored = context.stored as FunctionReference< + "query", + "internal", + Record, + { + streams: { streamVersion: number }[]; + events: { operationId: string }[]; + } + >; + const a = diagnosticApp({ + writes: [{ contextId: "depot", streamType: "counter" }], + executor: async (ctx, args) => + ctx.runMutation(add, { + tenantId: args.tenantId, + actor, + operation: args.operation, + input: { amounts: Array.from({ length: appended }, () => 1) }, + }), + }); + await a.grant(); + await a.t.run((ctx) => + ctx.runMutation(add, { + tenantId: call.tenantId, + actor, + operation: { + operationId: "seed", + causedBy: { kind: "command", commandType: "Seed" }, + }, + input: { amounts: Array.from({ length: 100 }, () => 1) }, + }), + ); + const before = await a.t.run((ctx) => ctx.runQuery(stored, {})); + expect(before.streams).toMatchObject([{ streamVersion: 100 }]); + expect(before.events).toHaveLength(100); + expect.soft(a.records()).toEqual([]); + const response = await a.t.mutation(internalRef, call); + const after = await a.t.run((ctx) => ctx.runQuery(stored, {})); + expect(after.streams).toMatchObject([{ streamVersion: 100 + appended }]); + expect(after.events).toHaveLength(100 + appended); + expect( + after.events.filter( + (event) => event.operationId === response.operationId, + ), + ).toHaveLength(appended); + expect.soft(a.records()).toEqual([ + { + ...fields, + kind: "applied", + replayed: false, + operationId: response.operationId, + eventsAppended: appended, + readModelRows: 0, + versions: response.versions, + }, + ]); + }); +} + +// Step 10's order is observable even though either wrong order could roll back cleanly. +test("convex-test: the receipt insert finishes before audit, and both finish before the diagnostic sink", async () => { + const writes: string[] = []; + const a = diagnosticApp({ + diagnosticSink: () => { + writes.push("diagnostic"); + }, + }); + await a.grant(); + await a.t.run((ctx) => + runPipeline( + { + ...ctx, + db: new Proxy(ctx.db, { + get(target, property, receiver) { + if (property === "insert") + return async (...args: unknown[]) => { + const result: unknown = await Reflect.apply( + target.insert, + target, + args, + ); + writes.push(String(args[0])); + return result; + }; + return Reflect.get(target, property, receiver); + }, + }), + }, + a.decl, + call, + ), + ); + expect( + writes.filter((table) => + ["receipts", "auditRecords", "diagnostic"].includes(table), + ), + ).toEqual(["receipts", "auditRecords", "diagnostic"]); + expect((await a.stored()).audits).toHaveLength(1); +}); From bc0183edbe53feadbecfacd2f36aac1500a8b89d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:02:40 +0200 Subject: [PATCH 132/155] design: 4c step 9 and the binding take the history projection Step 9 and readModelWrites call applyHistoryProjection once per command for each bound history read model with the entries' events and no version, and check the row count after that call; a history row counts once in the limit and in readModelRows. ReadModelBinding accepts a HistoryProjection, and binding once per source applies to history projections only. Co-Authored-By: Claude Opus 5.5 --- design/specs/command/command-declaration.sdp.md | 4 ++-- design/specs/command/command-pipeline.sdp.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/design/specs/command/command-declaration.sdp.md b/design/specs/command/command-declaration.sdp.md index 1eea025..461a160 100644 --- a/design/specs/command/command-declaration.sdp.md +++ b/design/specs/command/command-declaration.sdp.md @@ -43,7 +43,7 @@ The canonical schema is the declaration's `input` validator plus an optional ref - The declaration's `input` is a Convex `v.object` validator and its `refine` is an optional function over the validated value; together they are the one canonical schema, and no converter between schema systems is used (D12) - The declaration's `output` validator is the `result` of the executed member of the entry's `returns` validator, `commandResponseValidator(decl.output)`, whose replayed member carries `result: v.null()`; the wire type is closed per command and `result` is null exactly when `replayed` is true, because a receipt stores no result (D12, D4, D6, E-30) - The permission policy names the permission `authorize` checks and, optionally, how to derive the subject from the input (D11, D12) -- The read models the command maintains are listed in the declaration as bindings, each naming the read model, which carries its projection, and the context and stream type whose DTOs feed it, and the pipeline's step 9 writes them from the `streams` the executor returns; the executor writes none, and a declaration with no list maintains none (D8, D12, E-7) +- The read models the command maintains are listed in the declaration as bindings, each naming the read model, which carries its projection, and the context and stream type whose DTOs, or for a history projection whose events, feed it, and the pipeline's step 9 writes them from the `streams` the executor returns; the executor writes none, and a declaration with no list maintains none (D8, D12, E-7) - The executor receives the tenant, the actor, the minted `OperationRef` and the parsed input, captures the outside facts the decision needs, calls one context per context it names, with lists, and returns the kernel's committed outcome together with every context call's `streams` entries, or throws; a rejection is never a return value (D10, D12, D7, D3) - No DI container, decorator framework, reflection registry or module lifecycle; a class may group a module's declarations and holds no correctness-critical global state (D12) - Generation of the repetitive bindings starts only after two real modules show the pattern, is deterministic, and CI fails when regenerating changes a committed file (D12) @@ -63,7 +63,7 @@ What generation would emit, once two modules show the pattern: for each declarat - typePermissionPolicy: `type PermissionPolicy = { permission: string; subjectFrom?: (input: I) => SubjectRef }` (D11, D12) - typeExecutor: `type Executor = (ctx: MutationCtx, call: { tenantId: string; actor: Actor; operation: OperationRef; input: I }) => Promise>` where `OperationRef` is the one the pipeline minted at step 7, forwarded unchanged to every context call, and the executor's context calls are `ctx.runMutation` on component APIs with the facts it captured (D12, D10, D4, D3, E-26) - typeExecutorResult: `type ExecutorResult = CommittedOutcome & { streams: StreamDto[] }` where `CommittedOutcome` is the kernel's committed half of `Outcome` and `streams` concatenates the `streams` of every context outcome the executor received, the `StreamDto` of `spec:context.persistence-adapter`, so the pipeline's step 9 has every DTO, version and created flag it needs (D12, D4, D8, E-7, E-23) -- typeReadModelBinding: `type ReadModelBinding = { readModel: ReadModel; source: SourceRef }`, where `readModel` is the declaration `spec:application.projection-contract` pins and `source` selects which `streams` entries feed it; the binding repeats neither the read model's name nor its projection's version, a read model fed by several stream types is bound once per source, and the binding has no update function of its own (D8, D12, E-7, E-9) +- typeReadModelBinding: `type ReadModelBinding = { readModel: ReadModel | HistoryProjection; source: SourceRef }`, where `readModel` is a per-entity read model or a history projection as `spec:application.projection-contract` pins them and `source` selects which `streams` entries feed it; the binding repeats neither the read model's name nor its projection's version, a history projection is bound once per source stream type it folds while a per-entity read model is bound to its one source stream type, and the binding has no update function of its own (D8, D12, E-7, E-9) - typeAudit: `audit?: { kind: "security" | "business" }` asks the pipeline's step 10 to write one audit record through `writeAudit` of `spec:operations.baseline-operations`, which fails closed, after an applied command and after a business failure; absent, the default, the command writes none (D19, E-38, E-43) - typeDiagnosticSinkField: `diagnosticSink?: DiagnosticSink`, the sink type of `spec:operations.baseline-operations`, names where the pipeline writes the command's diagnostic lines; absent, the default, it is `consoleSink`; the production composition names none, and the fixture composition's `PlaceOrder` names the sink through which Sc ALL-1 breaks metrics and logging (D19, E-38, E-43) - typeAdmissionPolicy: `type AdmissionPolicy = (ctx: MutationCtx, call: PipelineCall) => Promise<{ admitted: true } | { admitted: false; code: "rateLimited" | "capacity"; retryAfterMs?: number }>`; a consumption it writes commits only with the command, so it counts committed intent (D6, E-32) diff --git a/design/specs/command/command-pipeline.sdp.md b/design/specs/command/command-pipeline.sdp.md index 3c987d5..e3bf15b 100644 --- a/design/specs/command/command-pipeline.sdp.md +++ b/design/specs/command/command-pipeline.sdp.md @@ -61,7 +61,7 @@ The pipeline is one code path with two registered entries per command. The publi - [non-blocking] Extension E-32: the doc places rate admission for new intent after the duplicate check and makes a rate or capacity refusal a retryable error, but names no admission mechanism; here admission is an optional per-command policy in the declaration, absent by default, evaluated inside the mutation so that a refusal that throws leaves no consumption behind; the flip side is that a consumption the policy writes commits only with the command, so the policy counts committed intent and cannot bound refused or unauthorized traffic, and a product that must bound refused traffic needs the dispatcher form of D7, where the parent survives the body's rejection (E-32, D4, D6, D7) - [non-blocking] Extension E-33: the doc lists the callers that need receipts and leaves UI commands to the client guarantee; here a public caller may still supply a request key to opt into a receipt, which Probe 1 showed a caller of `ConvexHttpClient` needs when it retries, and the internal entry always requires one (E-33, D6, Probe 1) - [non-blocking] Probe 4 showed on the pinned backend that the transaction limits are one budget across the parent and its component sub-transactions, so the per-command bounds are checked against one budget; `ctx.runMutation` accepts a `transactionLimits` option, the design passes none on a context call, and whether a call should reserve room with it for the work after it stays open (Probe 4, F13, S3) -- [non-blocking] Extension E-9: the doc says essential read models are updated inside the command and does not say from what or in what order; here step 9 writes a per-entity read model from the DTO of each matching `streams` entry, never from the events, checks `writes` first, then that each bound read model has a generation to write, then the count of the rows it writes against `limitReadModelWritesPerCommand`, each failure a plain error, and the library helpers that step 9 calls are generic over the application's data model; the owner has not ruled on any of it (E-9, E-7, E-8, E-45, D8, D9) +- [non-blocking] Extension E-9: the doc says essential read models are updated inside the command and does not say from what or in what order; here step 9 writes a per-entity read model from the DTO of each matching `streams` entry, never from its events, and a history read model from the events of every matching entry in one call per command, checks `writes` first, then that each bound read model has a generation to write, then the count of the rows it writes against `limitReadModelWritesPerCommand`, each failure a plain error, and the library helpers that step 9 calls are generic over the application's data model; the owner has not ruled on any of it (E-9, E-7, E-8, E-45, D8, D9) - [non-blocking] Extension E-34: step 1 bounds every text field a caller sets beside the input, the tenant ID, the request key, the correlation ID, the actor's fields and the fields of `causedBy`, in UTF-8 bytes as `limitCallText` lists them, as rejection `invalidInput`, and treats a declaration name above `limitIdLength` as a defect, a plain error, because these fields become fields of the receipt row or of every event's envelope; the doc says events stay small and names no number; the owner has not ruled (E-34, D6, D12, D19) ## Workflow @@ -86,7 +86,7 @@ The pipeline is one code path with two registered entries per command. The publi - Step 6, admit: for new intent only, the pipeline evaluates the declaration's admission policy; a refusal throws transient `rateLimited` or `capacity`, nothing is stored, and the same intent may be retried (D6, D4, Sc L1-9) - Step 7, gate and mint the operation: the pipeline reads the maintenance gate once through `assertWritable(ctx, scopesOfUseCase(tenantId, decl.writes))` of `spec:application.write-pause`, which refuses with the transient `writePaused` while a scope of this command is closed, the whole deployment, the tenant or a stream type it writes, or the deployment is in restore, and only then mints the whole `OperationRef`, the operation ID, `causedBy` from the call's `causedBy` argument when the internal entry received one and otherwise of kind `command` with the declaration's name, and the caller's optional correlation ID, and passes it to every context call so that events in different streams relate through it (Vocab: operation, D2, D9, D13, E-8, E-26, E-31) - Step 8, execute: the executor captures the outside facts the decision needs, from parent tables or configuration read in this transaction, and calls each context named by the declaration once through `ctx.runMutation` on the component's sanctioned operation, `components..operations.`, with tenant scope, actor, the operation reference, the command's input as lists and the captured facts; each call is an isolated sub-transaction that commits with the parent and rolls back on throw; a rejection thrown inside a context rolls its sub-transaction back and propagates to the boundary's outermost rethrow, and the executor returns the kernel's `CommittedOutcome` together with the `streams` entries every context call returned, never a rejection value (F2, F3, D3, D10, D7, E-23) -- Step 9, update read models: the pipeline first checks that every `streams` entry the executor returned names a context and stream type in the declaration's `writes`, throwing a plain error, a defect and a technical failure, when one does not, so the list the gate of step 7 was derived from is proven on the first native run; then, once for each read model its bindings name, however many bindings name it, it reads the generations to write through `generationsToWrite` and throws a plain error that names the command and the read model when there is none, and, for every binding, calls `applyProjection` of `spec:application.projection-contract` for each matching entry with the entry's DTO and version and the list unchanged, in the live-created mode when the entry's `created` flag is set and the live-updated mode otherwise, or `applyAggregate` or `applyHistoryProjection` for a binding of those forms, in the same transaction; it counts the rows those calls insert, replace or delete, a row in each generation once, and after each entry is applied throws a plain error when they pass `limitReadModelWritesPerCommand`, and the count is what the diagnostic record carries as `readModelRows`; the executor writes no read-model table itself (D8, D9, D12, Law 9, E-7, E-8, E-9, E-23, E-45) +- Step 9, update read models: the pipeline first checks that every `streams` entry the executor returned names a context and stream type in the declaration's `writes`, throwing a plain error, a defect and a technical failure, when one does not, so the list the gate of step 7 was derived from is proven on the first native run; then, once for each read model its bindings name, however many bindings name it, it reads the generations to write through `generationsToWrite` and throws a plain error that names the command and the read model when there is none, and, for every binding of a per-entity read model, calls `applyProjection` of `spec:application.projection-contract`, or `applyAggregate` for a binding of that form, for each matching entry with the entry's DTO and version and the list unchanged, in the live-created mode when the entry's `created` flag is set and the live-updated mode otherwise, and, once for each bound history read model, calls `applyHistoryProjection` with the events of every entry any of its bindings matches, in the order of the entries, with no version, the live-updated mode and the list unchanged, in the same transaction; it counts the rows those calls insert, replace or delete, a row in each generation once, a history row once however many entries fed it, and after each call throws a plain error when they pass `limitReadModelWritesPerCommand`, and the count, history rows included, is what the diagnostic record carries as `readModelRows`; the executor writes no read-model table itself (D8, D9, D12, Law 9, E-7, E-8, E-9, E-23, E-45) - Step 10, insert the receipt and the audit record: for a receipted call the pipeline inserts the thin receipt, outcome kind, operation ID, affected IDs derived from the stream versions, and the stream versions, in the same transaction as the lookup of step 5, so two concurrent identical requests serialize under optimistic concurrency and only one effect commits; for a declaration that sets `audit` it then calls `writeAudit` of `spec:operations.baseline-operations` with the record `auditRecordSources` of that Spec builds, whose failure throws and rolls the command back, a refused insert being a plain error and so a technical failure with nothing of the command stored (D6, D19, F1, Sc L1-3, E-43) - Step 11, return: the mutation builds the diagnostic record of the outcome from values it already holds and passes it to `emitDiagnostic` with the declaration's sink, which swallows every error, then returns the outcome kind, the result DTO, the operation ID, the affected IDs, the stream versions and `replayed` false (D4, D8, D2, D19, E-43) - At any step, a throw leaves the mutation uncommitted: the state and journal writes of every context sub-transaction, the read-model writes and the receipt are all rolled back, and nothing is stored as a business outcome (Law 2, D7, F2, Sc L1-2) @@ -114,9 +114,9 @@ The pipeline depends on three contracts it does not own: the context component's - contextCall: `await ctx.runMutation(components.inventory.operations.allocate, { tenantId, actor, operation, input: { orderId, lines }, facts })`, one call per context named by the declaration, lists in and the operation outcome out, the DTO with the versions and one `streams` entry per planned stream with its DTO, version, created flag and the events the call appended; the `args` shape is the one `spec:context.context-component` fixes, `operation` is the `OperationRef` of `spec:context.event-envelope` minted at step 7, and the return is the `OperationOutcome` of `spec:context.persistence-adapter` (D10, F2, E-22, E-23, E-26) - contextCallIsolation: each component call is a sub-transaction isolated from the other calls; a throw inside it rolls that sub-transaction back, and uncaught it rolls the parent back, so the parent never observes a half-applied context (F2, F3, S2) - rejectionRelay: a `ConvexError` carrying the kernel's `Rejection` thrown inside a component reaches the pipeline with its data intact under F14, which Probe 2 showed on the pinned backend; `normalizeThrown(error, decl.name, decl.rejections)` at the handler's outermost level rethrows it as the boundary's `RejectionData` when its code is a platform code or one the declaration's `rejections` names, and as a plain error otherwise; the fallback for a refuted F14, contexts returning the rejection as a value, is not needed (F14, Probe 2, D7) -- readModelWrites: parent-owned read-model tables written by the pipeline at step 9, one `applyProjection`, `applyAggregate` or `applyHistoryProjection` call per matching `streams` entry per binding the declaration lists, with the list `generationsToWrite` returns, each generation with its role; the executor holds the DTOs and the appended events only long enough to return them and writes no read-model table, so the declaration's list is the one wiring fact for read models (D8, D9, D12, E-7, E-9) +- readModelWrites: parent-owned read-model tables written by the pipeline at step 9, one `applyProjection` or `applyAggregate` call per matching `streams` entry per binding of those forms the declaration lists and one `applyHistoryProjection` call per bound history read model with the events of every entry its bindings match, with the list `generationsToWrite` returns, each generation with its role; the executor holds the DTOs and the appended events only long enough to return them and writes no read-model table, so the declaration's list is the one wiring fact for read models (D8, D9, D12, E-7, E-9) - writesCheck: `decl.writes` is proven at step 9 against the `streams` entries the executor returned, in memory and with no extra read; an entry outside the list throws a plain error, so a declaration whose `writes` understates its executor fails its first native run rather than passing a closed source scope at step 7 (E-8, E-38, Law 6) -- fnWriteReadModels: `writeReadModels(ctx: RegistryReader & RowWriter, commandType: string, bindings: readonly ReadModelBinding[], input: { tenantId: string; streams: readonly StreamDto[] }): Promise`, the read-model half of step 9 and the one function of the read-model library the pipeline calls, returning the number of read-model rows it inserted, replaced or deleted, a row in each generation once, the count it holds against `limitReadModelWritesPerCommand`; the pipeline passes its own ctx, which at run time is the parent's and holds the registry and the read model's table (D8, E-7, E-9) +- fnWriteReadModels: `writeReadModels(ctx: RegistryReader & RowWriter, commandType: string, bindings: readonly ReadModelBinding[], input: { tenantId: string; streams: readonly StreamDto[] }): Promise`, the read-model half of step 9 and the one function of the read-model library the pipeline calls, returning the number of read-model rows it inserted, replaced or deleted, a row in each generation once and a history row once however many entries fed it, the count it holds against `limitReadModelWritesPerCommand`; the pipeline passes its own ctx, which at run time is the parent's and holds the registry and the read model's table (D8, E-7, E-9) - helperGenerics: [extension] every library helper that takes the parent's own ctx or reads one of the parent's tables, `writeReadModels`, `activeGeneration` and `authorizeQuery` among them, is generic over the parent's data model, because a database reader is not covariant in its data model under the repository's compiler settings, so a helper typed with the library's tables alone refuses the parent's ctx (E-7, E-9, D11) - auditWrite: `writeAudit` of `spec:operations.baseline-operations` at step 10 for a declaration whose `audit` field is set, inside the transaction and after the receipt insert, given the `AuditRecordInput` that `auditRecordSources` of that Spec builds; its failure throws, the outermost catch passes the plain error through unchanged, the caller sees a technical failure and the command rolls back with nothing stored; a duplicate returns at step 5 before it and a throw at any earlier step skips it (D19, E-43, Sc ALL-1) - diagnosticEmit: `emitDiagnostic` of `spec:operations.baseline-operations` once per execution of an entry's handler, with the declaration's `diagnosticSink` or else `consoleSink`: at step 11 after every business write, at step 5 just before a duplicate is returned, and in the outermost catch through `relayFailure`; the record's fields come from the sources that `diagnosticSourcesReturned`, `diagnosticSourcesDuplicate` and `diagnosticSourcesThrown` of that Spec list, and `emitDiagnostic` swallows every error, so no diagnostic failure reaches the mutation (D19, E-43, Sc ALL-1) From c2d415e808d19812c907ad868e8d96ea2b040d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:05:19 +0200 Subject: [PATCH 133/155] build: integration the graph counts with the two Sc ALL-1 verifiers Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/design/README.md b/design/README.md index 7378535..0f2f647 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -232 specs · 5 packs · 131 anchors → 368 nodes · 1225 edges (0 errors, 0 warnings) +232 specs · 5 packs · 133 anchors → 370 nodes · 1227 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` From 207f3b71c901cfbb460093399dafa4c1c48e8a15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:05:19 +0200 Subject: [PATCH 134/155] build: integration a cut diagnostic line carries versionsOmitted when versions is empty fnDiagnosticLine places versionsOmitted on every line that measures more than limitDiagnosticBytes, counted in the fit; the code left it out when no entry was cut, which the pure proof refused. Co-Authored-By: Claude Opus 5.5 --- src/operations/diagnostic.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/operations/diagnostic.ts b/src/operations/diagnostic.ts index c0f7d10..81b46cd 100644 --- a/src/operations/diagnostic.ts +++ b/src/operations/diagnostic.ts @@ -47,15 +47,17 @@ const recordKeys = [ "eventsAppended", "readModelRows", ] as const; +// kept is undefined for the whole line. A cut line always carries versionsOmitted, 0 when versions +// is empty. function recordLine( record: DiagnosticRecord, - kept: number, + kept?: number, ): { line: string; bytes: number } { const fields: Record = {}; for (const key of recordKeys) if (record[key] !== undefined) fields[key] = record[key]; - const omitted = record.versions.length - kept; - if (omitted > 0) fields.versionsOmitted = omitted; + if (kept !== undefined) + fields.versionsOmitted = record.versions.length - kept; fields.versions = record.versions.slice(0, kept); const line = recordPrefix + JSON.stringify(fields); return { line, bytes: utf8Length(line) }; @@ -74,7 +76,7 @@ function cutLine(line: string): string { return line.slice(0, end); } export function diagnosticLine(record: DiagnosticRecord): string { - const whole = recordLine(record, record.versions.length); + const whole = recordLine(record); if (whole.bytes <= limitDiagnosticBytes) return whole.line; // Keep the longest prefix of versions for which the line, versionsOmitted counted, fits. for (let kept = record.versions.length - 1; kept >= 0; kept--) { From 7603ccf9925dd40ddf8128d93d03027d5d882ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:06:15 +0200 Subject: [PATCH 135/155] design: the close of round 3, five audit and diagnostic findings fixed Co-Authored-By: Claude Opus 5.5 --- design/reviews/consensus-ledger.json | 34 ++++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index 025b380..460ff60 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -2730,10 +2730,11 @@ ], "issue": "The audit path fails closed, and neither of its two writers can build the row the table requires. `fnWriteAudit` is `writeAudit(ctx: MutationCtx, record: AuditRecordInput)` and `AuditRecordInput` is defined nowhere. `tableAuditRecords` requires `operationId`, `commandType`, `subject: v.object({ contextId, streamType, streamId })`, `kind` and `decision: v.string()`. The pipeline's step 10 calls it 'for a declaration that sets `audit`', and the declaration gives only `audit?: { kind: \"security\" | \"business\" }`: `subject` has no source when the optional `permission.subjectFrom` is absent or when a use case touches many streams, as `PlaceOrder` with 101 does, `decision` has no stated values, and `causedBy` is `v.optional(v.string())` while the operation's cause is the `CausedBy` union. The second writer is the gate: write-pause says 'every close, resume and abort writes an audit record inside the same mutation and fails closed', and `closeScope` and `resumeScope` have a scope key, a reason and an actor, with no operation ID, no command type and no stream subject. baseline-operations says of the write that 'any failure, including a validator refusal or a document too large, throws and rolls the whole command back', so a wrong guess here refuses every audited command, and a gate close that cannot write its record refuses every paused rebuild.", "fix": "In `spec:operations.baseline-operations` define `AuditRecordInput` and give each field its source under E-43: `subject` is `decl.permission.subjectFrom(input)` when declared and otherwise the first affected ref, `decision` is the outcome kind, `causedBy` is the operation's cause in a stated string form, `requestKey` and `actor` come from the `PipelineCall`. Make the table fit an operator action: either `operationId`, `commandType` and `subject` become optional with a third `kind` of `operator` that carries `action` and `scopeKey`, or gate changes go to their own small table; say which in write-pause `fnCloseScope` and `fnResumeScope`. Repeat the field sources in the pipeline's `auditWrite` bullet so step 10 can be written from it.", - "status": "open", - "note": "S4 on paper, 2026-10-02. `spec:operations.baseline-operations` `typeAuditRecordInput`, `auditRecordSources`, `tableAuditRecords`, `tableOperatorAudit` and `fnWriteOperatorAudit` pin both records and the source of every field; `spec:application.write-pause` `fnCloseScope` and `fnResumeScope` write the gate's. The build must show at the compiled tier that `AuditRecordInput` is the table's document without `recordedAt`, and at convex-test step 10's record with each field from its source, both subject sources, and the fail-closed case with neither.", + "status": "fixed", + "note": "S4 on paper, 2026-10-02. `spec:operations.baseline-operations` `typeAuditRecordInput`, `auditRecordSources`, `tableAuditRecords`, `tableOperatorAudit` and `fnWriteOperatorAudit` pin both records and the source of every field; `spec:application.write-pause` `fnCloseScope` and `fnResumeScope` write the gate's. The build must show at the compiled tier that `AuditRecordInput` is the table's document without `recordedAt`, and at convex-test step 10's record with each field from its source, both subject sources, and the fail-closed case with neither. Close of round 3, 4a: every audit record field has its source in baseline-operations and command-pipeline step 10, written only for applied and business failure; proven at convex-test (tests/simulator/diagnostic.test.ts) and natively by Sc ALL-1's broken-audit-aborts.", "slice": "S4", - "detector": "TypeScript compiler" + "detector": "TypeScript compiler", + "unreviewed": true }, { "round": 3, @@ -2767,10 +2768,11 @@ ], "issue": "STATE lead 7, confirmed. `DiagnosticRecord.outcome` is '\"applied\" | \"businessFailure\" | \"rejection\" | \"technicalFailure\" | \"transientRefusal\"' with `errorCode?: string`, and the only emitter is the pipeline's 'Step 11, return: the mutation emits one diagnostic record', which runs after every business write on the success path. The outermost catch 'only rethrows', so a rejected, refused or failed command emits nothing, three of the five outcome values and `errorCode` are never produced, and D19's diagnosis 'by request key or operation ID' has no line for the commands that most need one. A replayed duplicate returns at step 5 and is not said to emit either. Three fields of the same record have no stated source: `durationMs`, although event-envelope says '`Date.now()` in the mutation, which Convex freezes for the whole function' and only the sweeper names `performance.now()`; `layer: number`; and `causedBy?: string` against the `CausedBy` union. `emitDiagnostic` is said to count swallowed errors by 'increments a counter in memory', which nothing can read back, while the ALL-1 example asserts 'the fixture's swallowed-error counter is one'.", "fix": "In `spec:operations.baseline-operations` either narrow `outcome` to the two committed kinds and drop `errorCode`, or say that the pipeline's outermost catch calls `emitDiagnostic` inside its own try before it rethrows, for rejections, transient refusals and other throws, and that a replay emits a record with `replayed` true; state which in the pipeline's `diagnosticEmit` bullet. Name `performance.now()` as the source of `durationMs`, define `layer`, give `causedBy` a string form, and replace the in-memory counter with something a test can observe, such as a second log line.", - "status": "open", - "note": "S1, 2026-10-01. Slice S1 built no audit and no diagnostic (ruling R2). S5, which builds audit and diagnostics, must show in a test which outcomes emit a record, a rejection, a transient refusal, a technical failure and a replay included, and the sources of `durationMs`, `layer` and `causedBy`.", + "status": "fixed", + "note": "S1, 2026-10-01. Slice S1 built no audit and no diagnostic (ruling R2). S5, which builds audit and diagnostics, must show in a test which outcomes emit a record, a rejection, a transient refusal, a technical failure and a replay included, and the sources of `durationMs`, `layer` and `causedBy`. Close of round 3, 4a: one diagnostic line per execution of a command entry's handler (step 11, step 5 for a duplicate, relayFailure for every throw), none for a call its validators refuse; proven at pure test and convex-test and natively by broken-metrics-never-abort.", "slice": "S5", - "detector": "pure unit test" + "detector": "pure unit test", + "unreviewed": true }, { "round": 3, @@ -3465,10 +3467,11 @@ ], "issue": "`writeAudit(ctx: MutationCtx, record: AuditRecordInput)` takes a type no Spec pins, and its two callers cannot fill the table from what they are said to have. `tableAuditRecords` requires `operationId`, `commandType`, `actor`, `subject: v.object({ contextId: v.string(), streamType: v.string(), streamId: v.string() })`, `kind` and `decision: v.string()`. The pipeline's step 10 calls it 'for a declaration that sets `audit`', and the declaration's `audit` is `{ kind: \"security\" | \"business\" }`; no bullet says which of a use case's affected streams is the one `subject` or what `decision` holds. write-pause says 'every close, resume and abort writes an audit record inside the same mutation and fails closed', through `closeScope` and `resumeScope`; a gate change has no operation ID, no command type and no stream subject, so it does not fit the required fields. One stale sentence sits beside this: `indexAuditByOperationUse` names 'the restore's receipt check' as a reader, and the restore's receipt check reads receipts and `byOperation` events, never audit records.", "fix": "In `spec:operations.baseline-operations` pin `AuditRecordInput` and say how step 10 fills it (for example `subject` from the declaration's `permission.subjectFrom(input)` or the first affected ref, `decision` from the outcome kind). For gate changes either make `operationId`, `commandType` and `subject` optional with a `scopeKey` field, or give the gate its own small audit table in `spec:application.write-pause`. Drop the restore from `indexAuditByOperationUse`.", - "status": "open", - "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: Closing or resuming a tenant/source gate supplies a scope, reason and actor but not the required operation ID, command type and stream subject, so the mandatory audit writer cannot construct its promised input. The build must show: Compiled AuditRecordInput and convex-test must represent command and gate actions, atomically store the declared audit evidence and roll back every mandatory action on audit failure; native backend acceptance must show the same refusal and unchanged business/gate state. S4 on paper, 2026-10-02. `spec:operations.baseline-operations` `typeAuditRecordInput`, `tableOperatorAudit`, `fnWriteOperatorAudit` and `indexOperatorAuditByScopeUse` pin both shapes, and `spec:application.write-pause` `fnGetGateAudit` reads the gate's by scope. The build must show at the compiled tier that each input type is its table's document without `recordedAt`, and at convex-test one `operatorAudit` record per gate change and a refused record leaving the gate unchanged.", + "status": "fixed", + "note": "the platform review of 2026-10-02, additional future work. Counterexample or trigger: Closing or resuming a tenant/source gate supplies a scope, reason and actor but not the required operation ID, command type and stream subject, so the mandatory audit writer cannot construct its promised input. The build must show: Compiled AuditRecordInput and convex-test must represent command and gate actions, atomically store the declared audit evidence and roll back every mandatory action on audit failure; native backend acceptance must show the same refusal and unchanged business/gate state. S4 on paper, 2026-10-02. `spec:operations.baseline-operations` `typeAuditRecordInput`, `tableOperatorAudit`, `fnWriteOperatorAudit` and `indexOperatorAuditByScopeUse` pin both shapes, and `spec:application.write-pause` `fnGetGateAudit` reads the gate's by scope. The build must show at the compiled tier that each input type is its table's document without `recordedAt`, and at convex-test one `operatorAudit` record per gate change and a refused record leaving the gate unchanged. Close of round 3, 4a: AuditRecordInput's sources pinned with step 10's order and the fail-closed rule; a refused insert is a plain error and nothing of the command survives, shown natively by broken-audit-aborts with the error naming auditRecords.", "slice": "S4", - "detector": "TypeScript compiler" + "detector": "TypeScript compiler", + "unreviewed": true }, { "round": 3, @@ -3878,10 +3881,11 @@ ], "issue": "baseline-operations' Design says `emitDiagnostic` runs 'inside a try and catch that swallows every error and increments a counter in memory', the rule says a sink failure 'is swallowed and counted', and the Sc ALL-1 example asserts 'that the fixture's swallowed-error counter is one'. No docs page says module state written inside a query or mutation is kept for a later function call: the runtimes page describes module scope as evaluated at deploy ('the system time used globally (outside of any function) is \"frozen\" at deploy time', a global `Math.random()` 'will not change between function runs'), and the components page says isolated environments mean 'writes to global variables ... are not shared' (docs.convex.dev/functions/runtimes and docs.convex.dev/components/understanding, read 2026-10-01). The docs are silent on whether a counter incremented in one mutation is visible to the next function the test calls, so the assertion rests on undocumented behavior and the count has no reader in production either. Separately, `DiagnosticRecord.durationMs` has no stated source. `Date.now()` 'will return the same result for the entirety of your function's execution', so a duration computed from it is always 0; `performance.now()` 'will increment inside mutations' and is 'fixed to same result during query function execution' (same runtimes page).", "fix": "In `spec:operations.baseline-operations` count a swallowed sink failure where it can be read: a second guarded `console.error` line with a fixed marker, read from the function log, which is what the example then asserts; drop 'a counter in memory'. State that `durationMs` is `performance.now()` at step 11 minus `performance.now()` at the handler's first line, valid in mutations only, and that a query's diagnostic carries no duration.", - "status": "open", - "note": "S1, 2026-10-01. Slice S1 built no diagnostic (ruling R2). S5, which builds diagnostics, must replace the in-memory counter with something a native test reads, such as a marked log line in the function log, and state and test the source of `durationMs`.", + "status": "fixed", + "note": "S1, 2026-10-01. Slice S1 built no diagnostic (ruling R2). S5, which builds diagnostics, must replace the in-memory counter with something a native test reads, such as a marked log line in the function log, and state and test the source of `durationMs`. Close of round 3, 4a: no duration is recorded, since Date.now() is fixed per execution and the completion record holds the execution time; the in-memory counter is replaced by one diagnostic gap line per lost record, counted from logLines natively and from a spy at convex-test.", "slice": "S5", - "detector": "native local-backend test" + "detector": "native local-backend test", + "unreviewed": true }, { "round": 3, @@ -4299,11 +4303,11 @@ ], "issue": "Final stream versions do not reveal how many events a command appended, and the diagnostic record has no count of read-model rows written.", "fix": "Record explicit bounded event and read-model write counters at their producers, or name backend measurements as the source. Test commands with equal final versions but different amounts of work.", - "status": "open", - "note": "the platform review of 2026-10-02, APPL-12. Evidence recorded by that read: Spec inspection. Counterexample: A stream ending at version 100 can have appended one or several events, and versions reveal neither the prior version nor the zero, one or two generation rows written. Spec evidence at 2e20b29: operations/baseline-operations.sdp.md:84 has versions but no work counters; :94,95 attribute the metrics to that record. confirmed against the Spec text by an independent check on 2026-10-02.", + "status": "fixed", + "note": "the platform review of 2026-10-02, APPL-12. Evidence recorded by that read: Spec inspection. Counterexample: A stream ending at version 100 can have appended one or several events, and versions reveal neither the prior version nor the zero, one or two generation rows written. Spec evidence at 2e20b29: operations/baseline-operations.sdp.md:84 has versions but no work counters; :94,95 attribute the metrics to that record. confirmed against the Spec text by an independent check on 2026-10-02. Close of round 3, 4a: eventsAppended from the streams' appended and readModelRows from writeReadModels' row count, proven at convex-test with one and two events at version 100 and zero, one and two generation rows.", "slice": "S5", "detector": "Pure tests must compare commands appending one and two events to an old stream at version 100 and projections writing zero, one and two generation rows; recorded metrics must equal actual work, with bounded detail truncation preserving the required counters.", - "unreviewed": false + "unreviewed": true }, { "round": "PR1", From 54473f305338e08f1b52463233f653680f909b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:07:39 +0200 Subject: [PATCH 136/155] build: hosted the four hosted examples in one driver tests/hosted/hosted-examples.test.ts binds the four hosted examples and runs them in order: Probe 7's retention reads the planting records and the parent's scheduler rows before anything is deployed or imported, then deploys the copy and plants; Probe 7's replacement in place exports, changes and imports on that copy with the run's own labels, counts the import in the planting records and lets every kept schedule finish; the production composition is deployed and places an order through an ordinary client; Probe 3 deploys the copy again and reads the usage around three sets of calls. Written and type-checked; no hosted deployment was contacted. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- ...ailed-runs.probe-7-hosted-retention.sdp.md | 2 +- harness/evidence.ts | 5 +- tests/hosted/examples.ts | 604 ++++++++++++++++++ tests/hosted/hosted-examples.test.ts | 226 +++++++ 5 files changed, 835 insertions(+), 4 deletions(-) create mode 100644 tests/hosted/examples.ts create mode 100644 tests/hosted/hosted-examples.test.ts diff --git a/design/README.md b/design/README.md index 7d73fa2..841ebf3 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -237 specs · 5 packs · 135 anchors → 377 nodes · 1239 edges (0 errors, 0 warnings) +237 specs · 5 packs · 139 anchors → 381 nodes · 1243 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.sdp.md b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.sdp.md index 02db1f9..fad6996 100644 --- a/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.sdp.md +++ b/design/specs/facts/f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.sdp.md @@ -24,7 +24,7 @@ And the run's own schedules are read in states {states: "success,failed,canceled ## Verification — executable -- Runs first in the hosted driver, before any deploy or import; it starts no local backend. When the deployment holds no planting record, the driver deploys the copy first and the run records that it read nothing. +- Runs first in the hosted driver, and reads before any deploy or import; it starts no local backend. After its read it deploys the copy, so that the planting module is the one of the run's commit; when the deployment holds no planting record, the run records that it read nothing. - A planted schedule's age is the reading's time less the `completedTime` its planting record holds, so it is known after the scheduler row is gone; the ages are read, never assumed from the schedule, because GitHub may delay or drop a scheduled run. - No age is bound: the ages are measured times, recorded in the run's evidence and on the fact and not asserted. The run reaches its boundary when it has planted its three schedules and recorded every earlier planting record, and its test result is failed otherwise; it is never skipped. A run's test result can be passed without showing a retention window, and the window stays open on F16 until records show it. - The records show the window when one shows a schedule present at one age and a later one shows it missing; with runs on Monday and Thursday the nominal ages are 3, 4, 7, 10, 11 and 14 days, and each recorded age uses the actual completion and reading times, so a bracket is no finer than those steps. diff --git a/harness/evidence.ts b/harness/evidence.ts index c89f934..fa08f84 100644 --- a/harness/evidence.ts +++ b/harness/evidence.ts @@ -42,11 +42,12 @@ const projectNames: readonly string[] = [ // native backend tier, not a tier of its own. export type Target = "local backend" | "hosted deployment"; // The deployment's answer to GET /api/v1/get_current_usage, with the time it was read. -export interface HostedUsage { +// A type and not an interface, so that a reading is itself a JSON value a measurement can hold. +export type HostedUsage = { readAt: string; seedStatus: string | null; response: JsonValue; -} +}; export interface HostedDeploy { composition: "fixture" | "production" | null; wallMs: number; diff --git a/tests/hosted/examples.ts b/tests/hosted/examples.ts new file mode 100644 index 0000000..4727185 --- /dev/null +++ b/tests/hosted/examples.ts @@ -0,0 +1,604 @@ +// The steps of the four hosted examples, which the hosted driver runs in order on the one hosted +// deployment. Each example asserts only on what its own run wrote, or on what the copy's table +// plantedSchedules says an earlier run planted. +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { getFunctionName } from "convex/server"; +import type { FunctionReturnType } from "convex/server"; +import { convexToJson } from "convex/values"; +import type { Value } from "convex/values"; +import { expect, inject, onTestFinished } from "vitest"; +import { + api as productionApi, + internal as productionInternal, +} from "../../example/convex/_generated/api.js"; +import { placeOrderPermission } from "../../example/convex/ordering.js"; +import { receiveStockPermission } from "../../example/convex/receiving.js"; +import { api as fixtureApi } from "../../fixture/convex/_generated/api.js"; +import type { CompletionRecord } from "../../harness/admin.js"; +import type { Backend } from "../../harness/backend.js"; +import { ordinaryClient } from "../../harness/clients.js"; +import type { HostedUsage } from "../../harness/evidence.js"; +import { hostedBackend, measure, required } from "../../harness/native.js"; +import type { HostedBackend } from "../../harness/native.js"; +import { archiveEntries } from "../native/backup-archive-entries.js"; +import { installOrderSummary, generations } from "../native/rebuild-install.js"; +import { + copyScopes, + hostedCopy, + plantSchedules, + scopeOptions, + type CopyScope, +} from "./copy.js"; +type Row = Record; +const measureValue = (name: string, value: unknown) => + measure(name, convexToJson(value as Value)); +const startedAt = () => inject("hostedRun").startedAt; +// A run's own name in the rows it writes, so that a kept dataset never mixes two runs. +export const runId = () => startedAt().replace(/[^0-9]/g, ""); +async function deployCopy(backend: HostedBackend) { + const copy = await hostedCopy(); + measure("temporary deployment", { + ...copy.added, + ...(await backend.admin.deployTemporary(copy.directory)), + }); +} + +// Probe 7's retention example. +export interface RetentionWorld { + backend?: HostedBackend; + readNothing?: boolean; + earlier?: Row[]; + observations?: { + scheduledId: string; + planted: string; + startedAt: string; + state: Value; + completedTime: Value; + imports: Value; + ageMs: number | null; + read: boolean; + }[]; + own?: Row[]; +} +export async function retentionDeployment(world: RetentionWorld) { + world.backend = await hostedBackend({ deploy: false }); +} +// Reads before any deploy or import, then deploys the copy and plants. A deployment that holds no +// planting record yet answers a read of the table as missing, and the run records that it read +// nothing. +export async function readThenPlant(world: RetentionWorld) { + const backend = required(world.backend, "the hosted backend"); + let earlier: Row[] = []; + try { + earlier = await backend.admin.readTable("plantedSchedules"); + world.readNothing = earlier.length === 0; + } catch (error) { + if (!String(error).includes('No table "plantedSchedules"')) throw error; + world.readNothing = true; + } + const scheduled = await backend.admin.readTable("_scheduled_functions"); + const readAt = Date.now(); + const present = new Set(scheduled.map((row) => String(row._id))); + world.earlier = earlier; + world.observations = earlier.map((record) => ({ + scheduledId: String(record.scheduledId), + planted: String(record.planted), + startedAt: String(record.startedAt), + state: record.state ?? null, + completedTime: record.completedTime ?? null, + imports: record.imports ?? null, + ageMs: + typeof record.completedTime === "number" + ? readAt - record.completedTime + : null, + read: present.has(String(record.scheduledId)), + })); + measureValue("planting records read", { + readAt: new Date(readAt).toISOString(), + readNothing: world.readNothing, + observations: world.observations, + }); + await deployCopy(backend); + await plantSchedules(backend, startedAt()); + const ownIds = new Set( + (await backend.admin.readTable("plantedSchedules")) + .filter((record) => record.startedAt === startedAt()) + .map((record) => String(record.scheduledId)), + ); + world.own = (await backend.admin.readTable("_scheduled_functions")).filter( + (row) => ownIds.has(String(row._id)), + ); + measureValue("own planted schedules", world.own); +} +export function earlierRecorded(world: RetentionWorld, recorded: boolean) { + const earlier = required(world.earlier, "the earlier planting records"); + const observations = required(world.observations, "the observations"); + expect( + observations.length === earlier.length && + observations.every( + (observation) => + typeof observation.read === "boolean" && + (typeof observation.ageMs === "number" || + observation.completedTime === null), + ), + ).toBe(recorded); +} +export function ownStates( + world: RetentionWorld, + states: string, + completedTime: boolean, +) { + const own = required(world.own, "the run's own schedules"); + expect(own.map((row) => (row.state as { kind: string }).kind).sort()).toEqual( + states.split(",").sort(), + ); + for (const row of own) + expect(typeof row.completedTime === "number").toBe(completedTime); +} + +// Probe 7's replacement in place. +// The copy mounts these components beside the parent; a component that another composition's +// deploy left unmounted is not among them. +export const mountedComponents = [ + "annex", + "annexClock", + "migrations", + "depot", + "yard", +]; +const scopeName = (scope: CopyScope) => scope ?? "parent"; +// Every user table of the parent and of every mounted component, read whole, keyed by +// scope/table, from the tables a backup archive holds. +async function storedData(backend: Backend, entries: string[]) { + const data: Record = {}; + for (const entry of entries) { + const match = + /^(?:_components\/([^/]+)\/)?([^_/][^/]*)\/documents\.jsonl$/.exec(entry); + if (match === null) continue; + const [, component, table] = match; + if (component !== undefined && !mountedComponents.includes(component)) + continue; + data[`${component ?? "parent"}/${table!}`] = await backend.admin.readTable( + table!, + component === undefined ? {} : { component }, + ); + } + return data; +} +async function schedulerRows(backend: Backend) { + const rows: Record = {}; + for (const scope of copyScopes) + rows[scopeName(scope)] = await backend.admin.readTable( + "_scheduled_functions", + scopeOptions(scope), + ); + return rows; +} +// The state kinds of the scheduler rows this run's labels name, in one scope. +async function ownKinds( + backend: Backend, + scope: CopyScope, + id: string, +): Promise { + return ( + await backend.admin.readTable("_scheduled_functions", scopeOptions(scope)) + ) + .filter((row) => JSON.stringify(row.args).includes(id)) + .map((row) => (row.state as { kind: string }).kind) + .sort(); +} +async function clearSchedulerTables(backend: Backend) { + for (const scope of copyScopes) + for (const table of [ + "schedulerData", + "schedulerReferences", + "schedulerEffects", + ]) + for (const row of await backend.admin.readTable( + table, + scopeOptions(scope), + )) + await backend.admin.writeTable( + table, + { delete: String(row._id) }, + scopeOptions(scope), + ); +} +async function createDocuments(backend: Backend, prefix: string) { + await backend.admin.run("depotRelay:createDocuments", { + tenantId: "hosted", + actor: { kind: "operator", id: "hosted-driver" }, + operation: { + operationId: prefix, + causedBy: { kind: "command", commandType: "fixture" }, + }, + input: { + documents: [1, 2].map((n) => ({ + documentId: `${prefix}-${n}`, + title: `${prefix} ${n}`, + })), + }, + }); + await backend.admin.run("markers:insert", { trial: prefix }); +} +export type InPlaceObservation = Awaited>; +export interface InPlaceWorld { + backend?: HostedBackend; + directory?: string; + observation?: InPlaceObservation; +} +// The copy is the one the retention example deployed, so the table plantedSchedules stays in the +// export. +export async function inPlaceDeployment(world: InPlaceWorld) { + world.backend = await hostedBackend({ deploy: false }); + const directory = await mkdtemp(join(tmpdir(), "libar-hosted-archive-")); + onTestFinished(() => rm(directory, { recursive: true, force: true })); + world.directory = directory; +} +// Runs the replacement in place with the run's own name in every label it schedules, and with +// pending reactions due a time after the start that leaves room for the export and the import. +export async function hostedInPlace( + backend: Backend, + directory: string, + options: { id: string; dueMs?: number }, +) { + const id = options.id; + const exportedLabel = `exported:${id}`; + const laterLabel = `after-export:${id}`; + // The deployment keeps earlier runs' scheduler tables. The reactions read the first data row, + // so the run starts them empty. + await clearSchedulerTables(backend); + const due = Date.now() + (options.dueMs ?? 180000); + await createDocuments(backend, `exported-${id}`); + for (const scope of copyScopes) { + await backend.admin.writeTable( + "schedulerData", + { insert: { value: "exported", held: true } }, + scopeOptions(scope), + ); + await backend.admin.run( + "scheduling:states", + { label: exportedLabel, due }, + scopeOptions(scope), + ); + } + for (const scope of copyScopes) + await expect + .poll(() => ownKinds(backend, scope, id), { timeout: 60000 }) + .toEqual(["canceled", "failed", "inProgress", "pending", "success"]); + await backend.admin.setEnvironment({ SCHEDULER_VALUE: "exported" }); + const path = join(directory, "backup-archive.zip"); + await backend.admin.exportBackupArchive(path); + const entries = await archiveEntries(path); + measure("backup archive entries", entries); + expect(entries).toContain("plantedSchedules/documents.jsonl"); + expect(entries.some((entry) => entry.includes("_scheduled_functions"))).toBe( + false, + ); + const exported = await storedData(backend, entries); + measureValue("exported data", exported); + + await createDocuments(backend, `after-export-${id}`); + for (const scope of copyScopes) { + const [row] = await backend.admin.readTable( + "schedulerData", + scopeOptions(scope), + ); + await backend.admin.writeTable( + "schedulerData", + { + patch: String(required(row, "the data row")._id), + fields: { value: "after-export" }, + }, + scopeOptions(scope), + ); + await backend.admin.run( + "scheduling:states", + { label: laterLabel, due }, + scopeOptions(scope), + ); + } + for (const scope of copyScopes) + await expect + .poll(() => ownKinds(backend, scope, id), { timeout: 60000 }) + .toEqual([ + "canceled", + "canceled", + "failed", + "failed", + "inProgress", + "inProgress", + "pending", + "pending", + "success", + "success", + ]); + await backend.admin.setEnvironment({ SCHEDULER_VALUE: "after-export" }); + const environment = await backend.admin.environment(); + const changedData = await storedData(backend, entries); + measureValue("data before the import", changedData); + for (const table of [ + "parent/markers", + "depot/streams", + "depot/events", + ...copyScopes.map((scope) => `${scopeName(scope)}/schedulerData`), + ]) + expect(changedData[table]).not.toEqual(exported[table]); + const before = await schedulerRows(backend); + await backend.admin.importBackupArchive(path); + const importedAt = Date.now(); + expect(importedAt).toBeLessThan(due); + const after = await schedulerRows(backend); + const restored = await storedData(backend, entries); + measureValue("restored data", restored); + const environmentAfter = await backend.admin.environment(); + // The planting records the import restored count it, read by the next run's retention example. + await backend.admin.run("planting:countImport", {}); + + const references = []; + for (const scope of copyScopes) { + const reference = required( + exported[`${scopeName(scope)}/schedulerReferences`]?.find( + (row) => row.label === exportedLabel, + ), + "the exported scheduler reference", + ); + const referenceId = String(reference.dispatchId); + const found = await backend.admin.run( + "scheduling:reference", + { id: referenceId, cancel: true }, + scopeOptions(scope), + ); + const canceled = ( + await backend.admin.readTable("_scheduled_functions", scopeOptions(scope)) + ).find((row) => row._id === referenceId); + references.push({ + scope: scopeName(scope), + id: referenceId, + found, + canceled, + }); + } + const reactions = []; + for (const scope of copyScopes) { + await expect + .poll( + async () => + ( + await backend.admin.readTable( + "schedulerEffects", + scopeOptions(scope), + ) + ).filter((row) => row.label === laterLabel).length, + { timeout: due - Date.now() + 60000, interval: 1000 }, + ) + .toBe(1); + const effects = await backend.admin.readTable( + "schedulerEffects", + scopeOptions(scope), + ); + reactions.push({ + scope: scopeName(scope), + kept: effects.find((row) => row.label === laterLabel), + effects, + }); + } + measureValue("kept reactions", { due, reactions }); + // Every kept schedule has run or is released before the example ends, so that no later example + // finds one pending or in progress. + for (const scope of copyScopes) + for (const row of await backend.admin.readTable( + "schedulerData", + scopeOptions(scope), + )) + await backend.admin.writeTable( + "schedulerData", + { patch: String(row._id), fields: { held: false } }, + scopeOptions(scope), + ); + for (const scope of copyScopes) + await expect + .poll( + async () => + (await ownKinds(backend, scope, id)).filter( + (kind) => kind === "pending" || kind === "inProgress", + ), + { timeout: 60000, interval: 1000 }, + ) + .toEqual([]); + return { + exported, + changedData, + restored, + before, + after, + environment, + environmentAfter, + references, + reactions, + due, + exportedLabel, + laterLabel, + }; +} + +// The production composition's acceptance on the hosted deployment. +export interface AcceptanceWorld { + backend?: Backend; + response?: FunctionReturnType; + records?: CompletionRecord[]; +} +export async function acceptanceDeployment(world: AcceptanceWorld) { + world.backend = await hostedBackend({ + composition: "production", + deploy: false, + }); +} +const tenantId = "hosted"; +export async function deployAndPlace( + world: AcceptanceWorld, + lines: number, + id: string, +) { + const backend = required(world.backend, "the hosted backend"); + await backend.admin.deploy(); + const subject = `hosted-${id}`; + for (const permission of [placeOrderPermission, receiveStockPermission]) + await backend.admin.run(getFunctionName(productionInternal.grants.grant), { + tenantId, + principalKind: "human", + principalId: `${backend.issuer.issuer}|${subject}`, + permission, + grantedBy: "hosted-driver", + }); + // The deployment keeps the order summary installed by an earlier run. + if ( + !(await generations(backend, "orderSummary")).some( + ({ generation }) => generation.state === "active", + ) + ) + await installOrderSummary(backend, "hosted-driver"); + const client = ordinaryClient(backend.url, { + token: await backend.issuer.token(subject), + }); + const items = Array.from({ length: lines }, (_, n) => ({ + stockItemId: `hosted-${id}-${n}`, + quantity: 1, + unitPrice: 100, + })); + await client.mutation(productionApi.receiving.receiveStock, { + tenantId, + input: { + items: items.map(({ stockItemId }) => ({ stockItemId, quantity: 2 })), + }, + }); + const mark = await backend.admin.logMark(); + world.response = await client.mutation(productionApi.ordering.placeOrder, { + tenantId, + requestKey: `hosted-${id}`, + input: { orderId: `hosted-${id}`, lines: items }, + }); + const identifier = getFunctionName(productionApi.ordering.placeOrder); + // The hosted log's retention is not published, so the run reads as it goes and records what it saw. + world.records = await backend.admin.completionsSince( + mark, + (records) => records.some((record) => record.identifier === identifier), + 60000, + ); + measure("function log after the command", { + completionRecords: world.records.length, + identifiers: world.records.map((record) => record.identifier), + }); +} +export function placeOrderRecords(world: AcceptanceWorld): CompletionRecord[] { + const identifier = getFunctionName(productionApi.ordering.placeOrder); + return required(world.records, "the function log").filter( + (record) => + record.identifier === identifier && record.componentPath === null, + ); +} + +// Probe 3's quota half. +export interface QuotaWorld { + backend?: HostedBackend; + reads?: number; + calls?: number; + seeds?: FunctionReturnType; + readings?: { set: string; readings: HostedUsage[] }[]; + added?: { set: string; functionCalls: number }[]; + settled?: { settled: boolean; reasons: string[] }; +} +const functionCalls = (usage: HostedUsage): number => { + const response = usage.response as { + metrics?: { functionCalls?: { usage?: { current_day?: number } } }; + }; + const value = response.metrics?.functionCalls?.usage?.current_day; + if (typeof value !== "number") + throw new Error("The usage reading holds no function calls for the day"); + return value; +}; +// Reads the usage until two readings in a row agree on the function calls, and keeps every reading. +async function settledReading( + backend: HostedBackend, + readings: HostedUsage[], +): Promise { + let last = await backend.admin.usage(); + readings.push(last); + for (let attempt = 0; attempt < 60; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 5000)); + const next = await backend.admin.usage(); + readings.push(next); + if (functionCalls(next) === functionCalls(last)) return next; + last = next; + } + throw new Error("Two usage readings in a row never agreed in 300 s"); +} +export async function quotaDeployment(world: QuotaWorld, reads: number) { + const backend = await hostedBackend({ deploy: false }); + await deployCopy(backend); + const seeds = await ordinaryClient(backend.url).mutation( + fixtureApi.readCost.seed, + {}, + ); + Object.assign(world, { backend, reads, seeds }); +} +export async function quotaSets(world: QuotaWorld, calls: number) { + const backend = required(world.backend, "the hosted backend"); + const client = ordinaryClient(backend.url); + const seeds = required(world.seeds, "the seeded documents"); + const reads = required(world.reads, "the read count"); + world.calls = calls; + world.readings = []; + world.added = []; + const start: HostedUsage[] = []; + let before = await settledReading(backend, start); + world.readings.push({ set: "start", readings: start }); + for (const [index, path] of ( + ["component", "helper", "component"] as const + ).entries()) { + for (let call = 0; call < calls; call++) + await client.mutation(fixtureApi.readCost.viaMutation, { + ...seeds, + path, + reads, + cacheBuster: Date.now() * 100 + index * calls + call, + }); + const readings: HostedUsage[] = []; + const after = await settledReading(backend, readings); + const set = `${index + 1}: ${path}`; + world.readings.push({ set, readings }); + world.added.push({ + set, + functionCalls: functionCalls(after) - functionCalls(before), + }); + before = after; + } + const all = world.readings.flatMap(({ readings }) => readings); + const [first, second, third] = world.added; + const reasons = [ + ...(second!.functionCalls === calls + ? [] + : [`the helper set added ${second!.functionCalls}, not ${calls}`]), + ...(first!.functionCalls === third!.functionCalls + ? [] + : [ + `the component sets added ${first!.functionCalls} and ${third!.functionCalls}`, + ]), + ...(all.every((reading) => reading.seedStatus === "complete") + ? [] + : ["a reading's seedStatus was not complete"]), + ...(all[0]!.readAt.slice(0, 10) === all.at(-1)!.readAt.slice(0, 10) + ? [] + : ["a UTC day boundary fell between the first and the last reading"]), + ]; + world.settled = { settled: reasons.length === 0, reasons }; + measure("function calls by set", { + calls, + reads, + added: world.added, + settled: world.settled, + readings: world.readings, + }); +} diff --git a/tests/hosted/hosted-examples.test.ts b/tests/hosted/hosted-examples.test.ts new file mode 100644 index 0000000..6fbaa90 --- /dev/null +++ b/tests/hosted/hosted-examples.test.ts @@ -0,0 +1,226 @@ +// The hosted driver: the four examples that run on the one hosted deployment, in this order, under +// npm run test:hosted alone. The retention example reads before anything is deployed or imported; +// the replacement in place runs on the copy it deployed; the production composition's acceptance +// deploys that composition; Probe 3 deploys the copy again and leaves it for the next run. +import { isDeepStrictEqual } from "node:util"; +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { bindExample } from "@libar-dev/software-delivery-protocol/vitest"; +import { expect, vi } from "vitest"; +import { probe3HostedFunctionCallsContract } from "../../generated/contracts/facts.f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls.contract.js"; +import { probe7HostedInPlaceImportContract } from "../../generated/contracts/facts.f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.contract.js"; +import { probe7HostedRetentionContract } from "../../generated/contracts/facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention.contract.js"; +import { hostedDeploymentAcceptsProductionCompositionContract } from "../../generated/contracts/platform.native-harness.hosted-deployment-accepts-production-composition.contract.js"; +import { required } from "../../harness/native.js"; +import { + acceptanceDeployment, + deployAndPlace, + earlierRecorded, + hostedInPlace, + inPlaceDeployment, + mountedComponents, + ownStates, + placeOrderRecords, + quotaDeployment, + quotaSets, + readThenPlant, + retentionDeployment, + runId, + type AcceptanceWorld, + type InPlaceWorld, + type QuotaWorld, + type RetentionWorld, +} from "./examples.js"; +const retentionAnchor = specTest({ + id: testAnchorId( + "test:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention", + ), + verifies: ref( + "spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention", + ), +}); +void retentionAnchor; +const inPlaceAnchor = specTest({ + id: testAnchorId( + "test:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import", + ), + verifies: ref( + "spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import", + ), +}); +void inPlaceAnchor; +const acceptanceAnchor = specTest({ + id: testAnchorId( + "test:platform.native-harness.hosted-deployment-accepts-production-composition", + ), + verifies: ref( + "spec:platform.native-harness.hosted-deployment-accepts-production-composition", + ), +}); +void acceptanceAnchor; +const quotaAnchor = specTest({ + id: testAnchorId( + "test:facts.f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls", + ), + verifies: ref( + "spec:facts.f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls", + ), +}); +void quotaAnchor; +vi.setConfig({ testTimeout: 900000 }); + +bindExample(probe7HostedRetentionContract, (): RetentionWorld => ({}), { + "the hosted deployment running the hosted driver's temporary copy of the fixture composition, which keeps the parent's schedules earlier runs planted and, in the table `plantedSchedules`, a planting record for each that names the run that planted it": + (world) => retentionDeployment(world), + "a run reads `plantedSchedules` and the parent's `_scheduled_functions` before it deploys or imports anything, then plants one succeeded, one failed and one canceled schedule": + (world) => readThenPlant(world), + "each planting record is recorded with its age and whether its schedule was read {recorded}": + (world, { recorded }) => earlierRecorded(world, recorded), + "the run's own schedules are read in states {states} each with a `completedTime` {completedTime}": + (world, { states, completedTime }) => + ownStates(world, states, completedTime), +}); + +type InPlace = InPlaceWorld & { + observation?: Awaited>; +}; +const observed = (world: InPlace) => + required(world.observation, "the replacement in place"); +bindExample(probe7HostedInPlaceImportContract, (): InPlace => ({}), { + "the hosted deployment running the hosted driver's temporary copy of the fixture composition, which adds the scheduler references and reactions of the local in-place example": + (world) => inPlaceDeployment(world), + "it exports business documents and five scheduler states with the deploy key, adds documents and schedules, then imports the backup archive in place before pending reactions are due": + async (world) => { + world.observation = await hostedInPlace( + required(world.backend, "the hosted backend"), + required(world.directory, "the archive directory"), + { id: runId() }, + ); + }, + "documents including ids and creation times in {scopes} equal the exported documents {equal}": + (world, { scopes, equal }) => { + const o = observed(world); + expect(scopes).toBe("parent and every mounted component"); + const read = new Set( + Object.keys(o.exported).map((key) => key.split("/")[0]), + ); + expect(read).toEqual( + new Set( + ["parent", ...mountedComponents].filter((scope) => read.has(scope)), + ), + ); + for (const scope of ["parent", "depot", "yard"]) + expect(read).toContain(scope); + expect(isDeepStrictEqual(o.restored, o.exported)).toBe(equal); + }, + "every scheduler row in each scope is unchanged {unchanged}": ( + world, + { unchanged }, + ) => { + const o = observed(world); + expect(Object.keys(o.after).sort()).toEqual(["depot", "parent", "yard"]); + for (const scope of Object.keys(o.after)) + expect(isDeepStrictEqual(o.after[scope], o.before[scope])).toBe( + unchanged, + ); + }, + "later business changes are gone {removed} and the destination environment is unchanged {environmentUnchanged}": + (world, { removed, environmentUnchanged }) => { + const o = observed(world); + const changed = Object.keys(o.changedData).filter( + (table) => !isDeepStrictEqual(o.changedData[table], o.exported[table]), + ); + expect(changed).toEqual( + expect.arrayContaining(["parent/markers", "depot/streams"]), + ); + for (const table of changed) + expect(isDeepStrictEqual(o.restored[table], o.exported[table])).toBe( + removed, + ); + expect(isDeepStrictEqual(o.environmentAfter, o.environment)).toBe( + environmentUnchanged, + ); + }, + "kept reactions read {value} no earlier than their scheduled time {onTime}": ( + world, + { value, onTime }, + ) => { + const o = observed(world); + expect(o.reactions).toHaveLength(3); + for (const reaction of o.reactions) { + const kept = required(reaction.kept, "the kept reaction"); + expect(kept.value).toBe(value); + expect(Number(kept._creationTime) >= o.due).toBe(onTime); + expect( + reaction.effects.filter((row) => row.label === o.exportedLabel), + ).toHaveLength(0); + } + }, + "each restored scheduler reference resolves to the kept row {found} and cancellation leaves state {state}": + (world, { found, state }) => { + const o = observed(world); + expect(o.references).toHaveLength(3); + for (const reference of o.references) { + expect( + isDeepStrictEqual( + reference.found, + o.before[reference.scope]!.find((row) => row._id === reference.id), + ), + ).toBe(found); + expect( + ( + required(reference.canceled, "the canceled row").state as { + kind: string; + } + ).kind, + ).toBe(state); + } + }, +}); + +bindExample( + hostedDeploymentAcceptsProductionCompositionContract, + (): AcceptanceWorld => ({}), + { + "the hosted deployment named by the run's deployment URL and deploy key, with the fixture issuer's variables set on it": + (world) => acceptanceDeployment(world), + "the run deploys the production composition, grants a subject with admin access, and an ordinary client carrying that subject's token places an order of {lines} line": + (world, { lines }) => deployAndPlace(world, lines, runId()), + "the deploy completes {deployed}": (world, { deployed }) => { + const backend = required(world.backend, "the hosted backend"); + expect(backend.facts().composition === "production").toBe(deployed); + }, + "the command's outcome is {outcome}": (world, { outcome }) => { + expect(required(world.response, "the answer")).toMatchObject({ + kind: outcome, + }); + }, + "the function log read with the deploy key holds the command's completion record {records}": + (world, { records }) => { + expect(placeOrderRecords(world)).toHaveLength(records); + }, + }, +); + +bindExample(probe3HostedFunctionCallsContract, (): QuotaWorld => ({}), { + "the hosted deployment running the hosted driver's temporary copy of the fixture composition, and a parent mutation that reads the same document {reads} times through a helper function or through a component query": + (world, { reads }) => quotaDeployment(world, reads), + "a client calls the parent mutation {calls} times through the component query, then {calls} times through the helper, then {calls} times through the component query again, and the run reads the deployment's function calls before and after each set of calls": + (world, { calls }) => quotaSets(world, calls), + "the function calls the helper set adds are {helperCallsPerParentCall} per parent call": + (world, { helperCallsPerParentCall }) => { + const [, helper] = required(world.added, "the sets"); + expect(required(helper, "the helper set").functionCalls).toBe( + helperCallsPerParentCall * required(world.calls, "the call count"), + ); + }, + "the function calls each component set adds are recorded, and the count is settled only when the two component sets add the same number": + (world) => { + const settled = required(world.settled, "the settlement"); + expect(settled.reasons, "the count is not settled").toEqual([]); + expect(settled.settled).toBe(true); + }, +}); From e1b497b0451befb4f4e84c2c8ae23e1cc4d8d3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:07:39 +0200 Subject: [PATCH 137/155] build: hosted the configuration and the workflow npm run test:hosted runs vitest.hosted.config.ts, whose one project, hosted, runs the driver with the hosted run's global setup; the root configuration includes none of it. .github/workflows/hosted.yml runs on dispatch and at 03:00 UTC on Monday and Thursday in the environment hosted, one run at a time, gives the key only to the steps that need it, masks its forms, and prints the captured log and uploads the record as hosted-evidence only after the record check finds no form. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/hosted.yml | 67 +++++++++++++++++++++ design/specs/platform/native-harness.sdp.md | 4 +- package.json | 5 +- tsconfig.json | 3 +- vitest.hosted.config.ts | 22 +++++++ 5 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/hosted.yml create mode 100644 vitest.hosted.config.ts diff --git a/.github/workflows/hosted.yml b/.github/workflows/hosted.yml new file mode 100644 index 0000000..6e60548 --- /dev/null +++ b/.github/workflows/hosted.yml @@ -0,0 +1,67 @@ +name: Hosted +# The native backend tier on the one hosted deployment, by npm run test:hosted. It runs on a +# schedule and on dispatch. GitHub runs a schedule from the default branch, and the environment +# hosted admits the default branch only, so a dispatch from another branch gets no key. No other +# workflow waits on this one, and its failure fails no other check. +on: + workflow_dispatch: + schedule: + - cron: "0 3 * * 1,4" +permissions: + contents: read +# One run at a time on the deployment; a queued run waits and no running run is cancelled. +concurrency: + group: hosted + cancel-in-progress: false +jobs: + hosted: + runs-on: ubuntu-latest + environment: hosted + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: npm + # The install runs before any step is given the key. + - run: npm ci + # GitHub masks the whole key; the part after the | and both URL-encoded forms are masked too. + - name: Mask the deploy key's forms + env: + HOSTED_DEPLOY_KEY: ${{ secrets.HOSTED_DEPLOY_KEY }} + run: | + node -e ' + const key = process.env.HOSTED_DEPLOY_KEY ?? ""; + const secret = key.slice(key.indexOf("|") + 1); + for (const form of [key, secret, encodeURIComponent(key), encodeURIComponent(secret)]) + if (form !== "") console.log("::add-mask::" + form); + ' + # The run's output goes to a file. The log is printed and the record uploaded only after the + # record check finds no form of the key in either. + - name: Native run on the hosted deployment + id: run + env: + HOSTED_DEPLOYMENT_URL: ${{ vars.HOSTED_DEPLOYMENT_URL }} + HOSTED_DEPLOY_KEY: ${{ secrets.HOSTED_DEPLOY_KEY }} + HOSTED_DEPLOY_KEY_NAME: ${{ vars.HOSTED_DEPLOY_KEY_NAME }} + HOSTED_PLAN: ${{ vars.HOSTED_PLAN }} + run: | + set +e + npm run test:hosted > "$RUNNER_TEMP/hosted.log" 2>&1 + echo "code=$?" >> "$GITHUB_OUTPUT" + - name: Record check + env: + HOSTED_DEPLOY_KEY: ${{ secrets.HOSTED_DEPLOY_KEY }} + run: | + shopt -s nullglob + records=(evidence/runs/hosted-*.json) + node scripts/hosted-record-check.mjs "$RUNNER_TEMP/hosted.log" "${records[@]}" + - name: The run's log + run: cat "$RUNNER_TEMP/hosted.log" + - uses: actions/upload-artifact@v7 + with: + name: hosted-evidence + path: evidence/runs/hosted-*.json + - name: The run's result + run: test "${{ steps.run.outputs.code }}" = "0" diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index b4e957d..cbb36ed 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -100,8 +100,8 @@ The harness is plain test support with no Convex function in it. The entries who - hostedSelection: [extension] a CLI call of a native run on a hosted deployment runs in the composition's project directory with the environment `PATH`, `HOME` set to a fresh directory, `TMPDIR` and `CONVEX_DEPLOY_KEY`, and no `--url` or `--admin-key`; `convex` 1.46.0 resolves a development deployment's deploy key to its URL and uses the key itself as the deployment's credential, in `node_modules/convex/src/cli/lib/deploymentSelection.ts` lines 714 to 749, and sends it as `Authorization: Convex` in `node_modules/convex/src/cli/lib/utils/utils.ts` lines 1373 to 1389; the local path keeps its arguments (E-59) - deployKeyActions: [extension] the deploy key allows exactly `deployment:deploy`, `deployment:env:view`, `deployment:env:write`, `deployment:data:view`, `deployment:data:write`, `deployment:functions:runInternalQueries`, `deployment:functions:runInternalMutations`, `deployment:functions:runInternalActions`, `deployment:logs:view`, `deployment:usage:view`, `deployment:backups:view`, `deployment:backups:create`, `deployment:backups:download` and `deployment:backups:import`, the actions the routes of `AdminAccess` and the CLI's export and import check in the upstream backend at `5c7cb5b`, and not `deployment:functions:actAsUser`, pause, usage limits, integrations or the audit log (E-59) - hostedRedaction: [extension] the secrets of a native run on a hosted deployment are the four forms of the deploy key, given to every child, every buffer and the reporter; the admin access built over a hosted deployment catches every rejection of its members, the HTTP client's raw response text and its `Invalid response` text among them, a fetch that rejects and a call that aborts, and rethrows it with those forms replaced; its HTTP and socket clients take a logger that replaces them; a CLI child's output is redacted on success as on failure and on an abort; the reporter redacts the serialized record before it writes it; the import upload token, the export download token, application data and the fixture issuer's private key are not among these secrets and are named as not covered, not scrubbed as if they were (E-59, E-15) -- hostedConfiguration: [extension] `npm run test:hosted` runs `vitest run --config vitest.hosted.config.ts`, whose one project, `hosted`, runs one driver file under `tests/hosted/` with its own global setup and the evidence reporter; the driver runs four examples in order: Probe 7's retention example before any deploy or import, Probe 7's replacement in place, the production composition's acceptance, and Probe 3's quota half; the first, second and fourth run on one temporary copy of the fixture composition that adds the modules under `tests/hosted/`, and the fourth leaves that copy deployed for the next run; when the deployment holds no earlier planting, the first run deploys the copy before the retention example and records that it read nothing; `npm run test:all` never includes it (E-59) -- hostedSchedule: [extension] the workflow `.github/workflows/hosted.yml` runs on `workflow_dispatch` and on a schedule at 03:00 UTC on Monday and Thursday; GitHub runs a schedule from the default branch, and the job runs in a GitHub environment named `hosted` that holds the key and admits the default branch only, so a dispatch from another branch gets no key; in one concurrency group that cancels no running job; it uploads the record as the artifact `hosted-evidence` only after `scripts/hosted-record-check.mjs`, reading the key from its own environment, finds no form of it in the record or the captured log, and no other workflow waits on it; until the workflow is on the default branch, the owner's machine runs `npm run test:hosted` with the variables in its shell, and its record names a developer as the one who ran it (E-59) +- hostedConfiguration: [extension] `npm run test:hosted` runs `vitest run --config vitest.hosted.config.ts`, whose one project, `hosted`, runs one driver file under `tests/hosted/` with its own global setup and the evidence reporter; the driver runs four examples in order: Probe 7's retention example before any deploy or import, Probe 7's replacement in place, the production composition's acceptance, and Probe 3's quota half; the first, second and fourth run on the one temporary copy of the fixture composition that `hostedProbeCopy` describes, and the fourth leaves that copy deployed for the next run; the retention example deploys the copy after its read and, when the deployment held no planting record, records that it read nothing; `npm run test:all` never includes it (E-59) +- hostedSchedule: [extension] the workflow `.github/workflows/hosted.yml` runs on `workflow_dispatch` and on a schedule at 03:00 UTC on Monday and Thursday; GitHub runs a schedule from the default branch, and the job runs in a GitHub environment named `hosted` that holds the key and admits the default branch only, so a dispatch from another branch gets no key; in one concurrency group that cancels no running job; the run's output goes to a file, and the job prints that captured log and uploads the record as the artifact `hosted-evidence` only after `scripts/hosted-record-check.mjs`, reading the key from its own environment, finds no form of it in the record or the captured log, and no other workflow waits on it; until the workflow is on the default branch, the owner's machine runs `npm run test:hosted` with the variables in its shell, and its record names a developer as the one who ran it (E-59) - hostedRedactionTest: [extension] one pure test with no network plants a key in the form of a development deploy key, gives the hosted admin access an injected fetch and a stub socket and runs a stub CLI script, each of which echoes the key in all four forms on every path, calls every member so that each fails, captures `process.stdout` and `process.stderr` and the reporter's output, writes a record into a temporary directory, and fails when any form appears in any of them (E-59) - hostedProbeCopy: [extension] the temporary copy of the fixture composition that the hosted driver deploys adds, from `tests/hosted/planting.ts` and the local in-place example's own scheduler module, the table `plantedSchedules` of planting records, one for each schedule a run planted in the parent, with its scheduler ID, the state and `completedTime` the run saw, the `startedAt` of the run that planted it and the count of backup archive imports since, the module `planting` that plants one succeeded, one failed and one canceled schedule and writes their planting records, and the scheduler references and reactions of the local in-place example in the parent and in the fixture contexts `depot` and `yard`; Probe 3's parent mutation is the fixture's own `readCost:viaMutation`, whose helper and component paths read the same document; the fixture composition itself gains nothing, as `probe7Narrowed` of `spec:facts.probe-plan` requires, and a later run knows a schedule's age after its scheduler row is gone (E-59) - compositionLayout: [extension] a composition is a name, a project directory and a functions directory; the Convex CLI reads `convex.json` and `package.json` from its working directory and takes no functions directory any other way, so the harness runs every CLI call in the composition's project directory; the fixture composition's project directory is the repository root, whose `convex.json` names `fixture/convex/`, and the production composition's is `example/`, whose `convex.json` names `convex/` and whose `package.json` is a symbolic link to the root's, so the repository has one `package.json` (E-15) diff --git a/package.json b/package.json index 75bf6df..8b33f28 100644 --- a/package.json +++ b/package.json @@ -16,14 +16,15 @@ "census": "sdp census", "sdp:build": "npx sdp build", "typecheck": "npm run sdp:build && tsc", - "lint": "eslint src fixture example harness tests scripts vitest.config.ts eslint.config.js", - "format:check": "prettier --check src fixture example harness tests scripts evidence/README.md README.md package.json tsconfig.json convex.json vitest.config.ts eslint.config.js .github/workflows/ci.yml .prettierrc.json", + "lint": "eslint src fixture example harness tests scripts vitest.config.ts vitest.hosted.config.ts eslint.config.js", + "format:check": "prettier --check src fixture example harness tests scripts evidence/README.md README.md package.json tsconfig.json convex.json vitest.config.ts vitest.hosted.config.ts eslint.config.js .github/workflows/ci.yml .github/workflows/hosted.yml .prettierrc.json", "test:types": "npm run sdp:build && vitest run --project types", "test:pure": "vitest run --project pure", "test:simulator": "vitest run --project simulator", "test:native": "npm run sdp:build && vitest run --project native", "test": "npm run sdp:build && vitest run --project types --project pure --project simulator", "test:all": "npm run sdp:build && vitest run", + "test:hosted": "npm run sdp:build && vitest run --config vitest.hosted.config.ts", "acceptance": "node scripts/acceptance.mjs", "codegen": "node scripts/codegen.mjs", "dev": "node scripts/dev.mjs", diff --git a/tsconfig.json b/tsconfig.json index f945406..8a154f2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,7 +20,8 @@ "example/**/*.ts", "harness/**/*.ts", "tests/**/*.ts", - "vitest.config.ts" + "vitest.config.ts", + "vitest.hosted.config.ts" ], "exclude": ["node_modules", "generated", "vendor"] } diff --git a/vitest.hosted.config.ts b/vitest.hosted.config.ts new file mode 100644 index 0000000..0fd3bae --- /dev/null +++ b/vitest.hosted.config.ts @@ -0,0 +1,22 @@ +import { defineConfig } from "vitest/config"; +// npm run test:hosted alone runs this configuration: the native backend tier on the one hosted +// deployment, selected by the four process variables its global setup reads. The root +// configuration does not include it, so npm test and npm run test:all never select a hosted target. +export default defineConfig({ + test: { + reporters: ["default", "./harness/evidence.ts"], + projects: [ + { + test: { + name: "hosted", + environment: "node", + include: ["tests/hosted/**/*.test.ts"], + globalSetup: ["./harness/hosted-run.ts"], + fileParallelism: false, + testTimeout: 900000, + hookTimeout: 120000, + }, + }, + ], + }, +}); From aa89f349b64934d91e56fa2d3670ae0427276596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:10:51 +0200 Subject: [PATCH 138/155] build: hosted the compiled check of the record and facts types Co-Authored-By: Claude Opus 5.5 --- design/README.md | 2 +- tests/types/hosted-record.test-d.ts | 76 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 tests/types/hosted-record.test-d.ts diff --git a/design/README.md b/design/README.md index 841ebf3..a92366b 100644 --- a/design/README.md +++ b/design/README.md @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -237 specs · 5 packs · 139 anchors → 381 nodes · 1243 edges (0 errors, 0 warnings) +237 specs · 5 packs · 140 anchors → 382 nodes · 1244 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/tests/types/hosted-record.test-d.ts b/tests/types/hosted-record.test-d.ts new file mode 100644 index 0000000..df4c958 --- /dev/null +++ b/tests/types/hosted-record.test-d.ts @@ -0,0 +1,76 @@ +import { + ref, + specTest, + testAnchorId, +} from "@libar-dev/software-delivery-protocol"; +import { expectTypeOf, test } from "vitest"; +import type { BackendFacts } from "../../harness/backend.js"; +import type { + HostedRunFacts, + HostedUsage, + JsonValue, + NativeRunRecord, + NativeTestEntry, +} from "../../harness/evidence.js"; +import type { HostedAccess, HostedTarget } from "../../harness/hosted.js"; +import type { HostedBackend } from "../../harness/native.js"; +// Binds this file to the harness Spec, whose record and facts types for a native run on a hosted +// deployment it checks at the compiled tier. +const anchor = specTest({ + id: testAnchorId("test:platform.native-harness.hosted-record-types"), + verifies: ref("spec:platform.native-harness"), +}); +void anchor; + +test("compiled: a run record names its target apart from its tier and holds the hosted facts or null", () => { + expectTypeOf().toEqualTypeOf<"native">(); + expectTypeOf().toEqualTypeOf< + "local backend" | "hosted deployment" + >(); + expectTypeOf< + NativeRunRecord["hosted"] + >().toEqualTypeOf(); + expectTypeOf().toEqualTypeOf< + "types" | "pure" | "simulator" | "native" | "hosted" + >(); + expectTypeOf().toEqualTypeOf<{ + deployment: string; + keyName: string; + statedPlan: string; + ranBy: "continuous integration" | "developer"; + cliVersion: string; + backendVersion: string | null; + usageBefore: HostedUsage | null; + usageAfter: HostedUsage | null; + windowCrossed: boolean; + deploys: { + composition: "fixture" | "production" | null; + wallMs: number; + }[]; + }>(); + expectTypeOf().toEqualTypeOf<{ + readAt: string; + seedStatus: string | null; + response: JsonValue; + }>(); +}); + +test("compiled: a backend's facts say its target, and on a hosted deployment no executable and a kept dataset", () => { + expectTypeOf().toEqualTypeOf< + "local backend" | "hosted deployment" + >(); + expectTypeOf().toEqualTypeOf< + "empty at start" | "kept from earlier runs" + >(); + expectTypeOf().toExtend(); +}); + +test("compiled: the hosted target holds no deploy key, and the hosted backend's admin access reads usage", () => { + expectTypeOf().toEqualTypeOf< + "url" | "deployment" | "keyName" | "statedPlan" + >(); + expectTypeOf().toEqualTypeOf(); + expectTypeOf>().toEqualTypeOf< + Promise + >(); +}); From 7c70057285f029c84fd47ce0a7cc7785f470ec22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:32:39 +0200 Subject: [PATCH 139/155] design: close fold-in F1 F4 F5 F6 F7 F8 F9 F11 F12 F13 F25 F34 The acceptance contract takes the ruled pass of the first experiment, the README counts twenty-four rows, restore records the enumeration of a stream awaiting a baseline as an open question and inventories it, a parent query relays writePaused unchanged, entry names every kind of entry, the aggregate form joins ReadModelBinding, the f19 ID is corrected, the tier and work words are fixed, and "backup archive" names the exported artifact. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 10 ++++----- .../first-experiment.order-of-1-line.sdp.md | 2 +- .../first-experiment.order-of-10-lines.sdp.md | 2 +- ...first-experiment.order-of-max-lines.sdp.md | 2 +- ...ventory-example.cancel-placed-order.sdp.md | 2 +- ...ample.cancel-second-context-rejects.sdp.md | 2 +- design/specs/application/rebuild.sdp.md | 2 +- ...tore.restore-representative-dataset.sdp.md | 2 +- design/specs/application/restore.sdp.md | 21 ++++++++++--------- design/specs/application/write-pause.sdp.md | 10 ++++----- .../specs/command/command-declaration.sdp.md | 2 +- design/specs/command/outcome-boundary.sdp.md | 6 +++--- design/specs/context/context-component.sdp.md | 2 +- ...uled-functions.probe-7-fresh-import.sdp.md | 4 ++-- ...d-functions.probe-7-in-place-import.sdp.md | 4 ++-- ...exclude-pending-scheduled-functions.sdp.md | 2 +- .../obligations/retention-and-restore.sdp.md | 4 ++-- .../specs/platform/acceptance-contract.sdp.md | 2 +- .../transactional-domain-platform.sdp.md | 4 ++-- ...led-functions.probe-7-fresh-import.test.ts | 2 +- ...-functions.probe-7-in-place-import.test.ts | 2 +- 21 files changed, 45 insertions(+), 44 deletions(-) diff --git a/design/README.md b/design/README.md index 6c3772f..b2a8701 100644 --- a/design/README.md +++ b/design/README.md @@ -101,7 +101,7 @@ Layer 1, context side. A bounded context is one Convex component with `streams` Layer 1, parent side. Every command runs through one top-level mutation with eleven steps: parse, authenticate, establish scope, where the public entry authenticates after Convex's argument validators and before step 1's key, bound and refinement checks, authorize, look up the receipt, admit, read the maintenance gate and mint the whole `OperationRef`, execute the contexts through `ctx.runMutation` on their operations with the facts the executor captured, update read models from the `streams` the executor returned, insert the receipt and the audit record, emit a diagnostic and return. The only catch is an outermost rethrow that gives a context's bare `Rejection` and the pipeline's own rejections one wire shape, and turns a context's code that is neither a platform code nor in the declaration's `rejections` into a technical failure. Each command has one declaration and two static exports: `publicCommand` (a `mutation`, namespace fixed to `public`, reads `ctx.auth`) and `internalCommand` (an `internalMutation` that trusted server code calls with `actor`, `namespace` and a required `requestKey`); the doubled function count and the single-export alternative it beat are recorded under E-7. Receipts are thin, keyed by four indexed fields, guarded by a SHA-256 fingerprint that excludes captured facts, read and inserted in the same transaction; a rejection or transient refusal writes none, and the second UI submit of a create is answered with `entityExists`, not an idempotent success (E-39). The declaration's `bounds` is the one input bound, checked at step 1, and its `writes` names the stream types the executor may write. The internal entry takes an optional `causedBy`, which is how a reaction's derived command names its publication event (E-31), and a derived command carries the `worker` namespace (E-6). The actor, tenant scope, caller namespace, the worker's `Authority` and the grants table are defined once in [`actor-and-scope`](specs/command/actor-and-scope.sdp.md) and reused by the envelope, the receipts table and the obligation record; grants are written by the plain helpers `insertGrant` and `revokeGrant`, over which the application registers its own internal mutations (E-37). -Layer 2. A use case calls each context once with a list, applies the cross-context rule to the DTOs it gets back, and returns the union of stream versions with every context call's `streams` entries; the pipeline's step 9 writes the read models the declaration lists from those entries through one `applyProjection` shared with rebuild, and a rejection or throw from the second context rolls back the first. Rebuild is a new numbered generation: live commands write it too, backfill batches of at most 100 entities, capped by the context's byte-derived page bound, write only where the target is missing or older, verification precedes a one-write switch, and the old generation stays for 7 days. Cross-stream history views rebuild under the write pause, one gate document the pipeline reads for every command, with the generation's fence for background writers. Such a view is a `HistoryProjection` that folds events into its row; its paused rebuild closes the view's source stream types, so every writer of those streams is held whether or not it maintains the view, and its backfill pages each source's history. Restore closes writers through `MAINTENANCE_MODE`, restores the snapshot, runs four checkpointed checks (journal, domain, read model, receipt), reopens only on zero findings, and only then hands over to the durable profile's schedule rebuild. Audit is written inside the transaction and fails closed; diagnostics go through a sink that never throws. The first experiment is Orders and Inventory with `PlaceOrder`, `CancelOrder`, the order summary and the order allocation history view that Sc L2-6 builds; its six cost targets are constraint Specs, and its native runs authenticate ordinary clients with tokens signed by a fixture issuer that the deployment's environment variables name, the one named difference from production (E-13). +Layer 2. A use case calls each context once with a list, applies the cross-context rule to the DTOs it gets back, and returns the union of stream versions with every context call's `streams` entries; the pipeline's step 9 writes the read models the declaration lists from those entries through one `applyProjection` shared with rebuild, and a rejection or throw from the second context rolls back the first. Rebuild is a new numbered generation: live commands write it too, backfill batches of at most 100 entities, capped by the context's byte-derived page bound, write only where the target is missing or older, verification precedes a one-write switch, and the old generation stays for 7 days. Cross-stream history views rebuild under the write pause, one gate document the pipeline reads for every command, with the generation's fence for background writers. Such a view is a `HistoryProjection` that folds events into its row; its paused rebuild closes the view's source stream types, so every writer of those streams is held whether or not it maintains the view, and its backfill pages each source's history. Restore closes writers through `MAINTENANCE_MODE`, restores the backup archive, runs four checkpointed checks (journal, domain, read model, receipt), reopens only on zero findings, and only then hands over to the durable profile's schedule rebuild. Audit is written inside the transaction and fails closed; diagnostics go through a sink that never throws. The first experiment is Orders and Inventory with `PlaceOrder`, `CancelOrder`, the order summary and the order allocation history view that Sc L2-6 builds; its six cost targets are constraint Specs, and its native runs authenticate ordinary clients with tokens signed by a fixture issuer that the deployment's environment variables name, the one named difference from production (E-13). Layer 3. An obligation is one recorded promise that commits with the business change and whose first dispatch is scheduled in the same transaction. The module lives in the parent as a library plus the `obligations` and `obligationRepairs` tables. A command's static reactions are the `createObligation` calls of its executor body, not a field of its declaration, and the record stores the one `Authority` shape of the actor and scope contract. A local reaction is a scheduled wrapper that runs the body as a nested mutation, so the effect and its completion commit together or the wrapper records the attempt after the body failed; a body refused by the write pause or a capacity policy is deferred without counting an attempt. A cron sweeper reads `_scheduled_functions`, whose `state.kind` is a lowercase literal, to tell a queued dispatch from a dead one, rearms at most five times, and escalates. External effects claim in a mutation, call in an action that makes exactly one provider call, and settle in a second mutation under one of four repetition policies; a stale worker's late evidence is kept and reconciled. Retention never erases unresolved work; restore starts with `OBLIGATIONS_DISPATCH` off and rebuilds schedules from the obligations after the transactional restore has reopened writers. All of it is gated on Probe 7 through the [`do-nothing-check`](specs/obligations/do-nothing-check.sdp.md) decision, made once for the durable profile and not per reaction, and every numeric default in the layer is provisional until the first experiment, which each Layer 3 Spec records against decision method rule 4. @@ -200,7 +200,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-14 | [`platform.acceptance-contract`](specs/platform/acceptance-contract.sdp.md) | The compiled tier, which the doc's four tiers lack: a type test of a composition's generated `api`, a typed caller that states the change and must fail `tsc`, run by the type check with no backend; Sc L2-4 is its one scenario | | E-15 | [`platform.native-harness`](specs/platform/native-harness.sdp.md) | The native harness: the admin key is for setup and reading, never for a caller under test; two compositions, fixture and production, each a name, a project directory and a functions directory, the production composition's project directory being `example/`, with the generated files of both committed; faults only through what the fixture composition supplies, and no test hook in the shared libraries; assertions only on what a client or admin access can read, what a command read and wrote taken from the `usageStats` of its top-level completion record, and a refusal shown with its own error, nothing stored and the refused thing existing, shown by an ordinary client where one may call it and with admin access otherwise; the fixture action `nonUiCaller:send` as the fixture composition's stand-in for trusted server code; four controls (the backend process, the clients, the environment variables, fixture functions) and no clock or scheduler switch; the production-configuration run of Sc L2-9 is the scenarios routed to the production composition; a pass claim cites a record kept in the repository | | E-16 | [`facts.probe-plan`](specs/facts/probe-plan.sdp.md) | How a probe is written and recorded: example Specs that verify the fact, bound values as the expectation written before the first run, a different first result recorded on the fact and the example then bound to what the release does, a run that must reach its boundary, sizes recorded and not asserted, and what a local backend cannot answer left open on the fact | -| E-17 | [`platform.acceptance-contract`](specs/platform/acceptance-contract.sdp.md), [`application.first-experiment`](specs/application/first-experiment.sdp.md) | The acceptance check: the required scenarios generated from the graph, one per example of a required row with its tier, composition and verifiers, read against one run's record; a scenario is passed, failed, absent or missing, and the whole passes only when every one passed on a record of a passed run on a clean tree; `acceptanceRows` of `application.first-experiment` names the first experiment's twenty-three rows | +| E-17 | [`platform.acceptance-contract`](specs/platform/acceptance-contract.sdp.md), [`application.first-experiment`](specs/application/first-experiment.sdp.md) | The acceptance check: the required scenarios generated from the graph, one per example of a required row with its tier, composition and verifiers, read against one run's record; a scenario is passed, failed, absent or missing, and the whole passes only when every one passed on a record of a passed run on a clean tree; `acceptanceRows` of `application.first-experiment` names the first experiment's twenty-four rows | | E-20 | [`kernel.decider-contract`](specs/kernel/decider-contract.sdp.md), [`kernel.domain-kernel`](specs/kernel/domain-kernel.sdp.md) | The concrete generics, `DomainEvent`, `DecisionActor` as the kernel's structural view of the actor, `DecisionContext { now, actor, facts }`, `DecideResult`, `Invariant`, `Transitions` and `transition`; the kernel imports nothing from Convex and nothing from the command family | | E-21 | [`kernel.outcome-model`](specs/kernel/outcome-model.sdp.md) | `Outcome`, `Rejection`, `CommittedOutcome`, `AffectedRef`; a committed outcome carries at least one event; the reserved codes `staleVersion`, `operationTooLarge` and `entityExists`, the last raised by the adapter for a create planned at expected version 0 whose stream exists | | E-22 | [`context.context-component`](specs/context/context-component.sdp.md) | One registered mutation per sanctioned operation via `defineOperation` with args `{ tenantId, actor, operation, input, facts }`; `contextId` as a code constant equal to the mount name; 256 streams, 800 written documents and 8 MiB of stream documents read or written per call, derived from each stream type's byte budget; `convex-helpers` as the component's dependency; the query and maintenance surface | @@ -220,7 +220,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-39 | [`command.idempotency-and-receipts`](specs/command/idempotency-and-receipts.sdp.md) | The second UI submit of a create is answered with the rejection `entityExists`, not an idempotent success, because the UI path writes no receipt to replay from and the client holds the entity ID | | E-40 | [`application.rebuild`](specs/application/rebuild.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | A batch is one mutation with one cursor on the progress row, at most 100 entities capped by the row budget and by the context's byte-derived page bound, a design number not attributed to the component; a self-scheduled internal mutation drives every batch, the backfill, the verify pass and the purge, and the rebuild uses `@convex-dev/migrations` for nothing; one batch names one tenant, taken from the tenant list; a fence that every interrupt, resume, rollback, abort and purge start raises | | E-41 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | The caller-driven import command with an `imports` progress row; the input bound itself is the declaration's `bounds.maxItems` under E-38 | -| E-42 | [`application.restore`](specs/application/restore.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | Four checks (journal, domain, read model, receipt) as checkpointed batches, the stream check bounded through the `rebuild` query's `maxEvents` so its worst case is one fold, and a refused fold counted as a journal finding; a fifth step that inventories the closed gate entries, the generations in flight and the running imports the snapshot restored; the `restoreRuns` record; an accepted branch that resumes every stranded generation, proven by the generation row's fence; `MAINTENANCE_MODE=restore` as the writer door, read by the write pause's check before the gate document and cleared before the durable profile's schedule rebuild runs; a failed check keeps the whole deployment closed with no override; the drill before every release, its cadence reconsidered on the first drill's measured check times | +| E-42 | [`application.restore`](specs/application/restore.sdp.md), [`application.generation-registry`](specs/application/generation-registry.sdp.md) | Four checks (journal, domain, read model, receipt) as checkpointed batches, the stream check bounded through the `rebuild` query's `maxEvents` so its worst case is one fold, and a refused fold counted as a journal finding; a fifth step that inventories the closed gate entries, the generations in flight and the running imports the backup archive restored; the `restoreRuns` record; an accepted branch that resumes every stranded generation, proven by the generation row's fence; `MAINTENANCE_MODE=restore` as the writer door, read by the write pause's check before the gate document and cleared before the durable profile's schedule rebuild runs; a failed check keeps the whole deployment closed with no override; the drill before every release, its cadence reconsidered on the first drill's measured check times | | E-43 | [`operations.baseline-operations`](specs/operations/baseline-operations.sdp.md) | The `auditRecords` table that fails closed, with `AuditRecordInput` and the source of each field in the command; the `operatorAudit` table for the gate changes an operator entry makes; Convex's own audit logging recorded as the do-nothing option; a 4 KiB diagnostic record through a sink that never throws; the `personalData` reference table | | E-44 | [`application.parent-use-cases`](specs/application/parent-use-cases.sdp.md) | `BusinessFailurePolicy` (`carry` or `reject`) per context call, applied in the executor body through the pinned helper `carryOrReject` | | E-45 | [`application.read-models`](specs/application/read-models.sdp.md) | At most 8 component reads per parent query and 4 read-model rows written per command, counted as the rows step 9 inserts, replaces or deletes and checked after each `streams` entry is applied, above which step 9 throws a plain error | @@ -234,7 +234,7 @@ An extension is a design claim the doc does not make. Each one is three things t | E-53 | [`obligations.operator-operations`](specs/obligations/operator-operations.sdp.md) | The five operations as authorized parent functions; cancel and reconcile refused while a lease is live; reconcile mints a reconcile attempt that the claim mutation dispatches, or accepts supplied provider evidence from needs attention | | E-54 | [`obligations.fan-out-and-chains`](specs/obligations/fan-out-and-chains.sdp.md) | 16 obligations per publishing transaction, 500 per fan-out page, a causation depth of 8; a command's static reactions are the `createObligation` calls of its executor body, not a declaration field | | E-55 | [`obligations.retention-and-restore`](specs/obligations/retention-and-restore.sdp.md) | A completed-work horizon of the larger of 30 days and the longest provider-key validity; compaction after 7 days; 500 rows per retention run | -| E-56 | [`obligations.retention-and-restore`](specs/obligations/retention-and-restore.sdp.md) | External dispatch switched by the `OBLIGATIONS_DISPATCH` environment variable, read only in the parent; the cursor-driven `rebuildSchedules` operation, which refuses while `MAINTENANCE_MODE` is `restore`; the switch order, dispatch off before the snapshot and on last, after the transactional restore has reopened writers | +| E-56 | [`obligations.retention-and-restore`](specs/obligations/retention-and-restore.sdp.md) | External dispatch switched by the `OBLIGATIONS_DISPATCH` environment variable, read only in the parent; the cursor-driven `rebuildSchedules` operation, which refuses while `MAINTENANCE_MODE` is `restore`; the switch order, dispatch off before the backup archive is restored and on last, after the transactional restore has reopened writers | | E-57 | [`obligations.record-contract`](specs/obligations/record-contract.sdp.md), [`obligations.sweeper`](specs/obligations/sweeper.sdp.md) | Three deployment-wide indexes lead with `status`, not `tenantId`; a recorded deviation from the review rubric's item 25 | | E-58 | [`effects.claim-call-settle`](specs/effects/claim-call-settle.sdp.md), [`effects.external-effects`](specs/effects/external-effects.sdp.md) | The lease is the runtime's action timeout plus 5 minutes; the provider key is `tenantId:effectKey`; ten late reports kept; a reconcile returns the same report union; plus three bounded repeats of a thrown settle call with a duplicate rule in the settle, so a repeat is not an attempt under Law 12 | | E-59 | [`platform.native-harness`](specs/platform/native-harness.sdp.md) | A native run on a hosted deployment: the same harness, run record and compositions on one hosted development deployment in a team and project of their own, selected only by `npm run test:hosted` from process variables, one run at a time, from a workflow whose failure fails no other check; the deploy key reaches a CLI child only as `CONVEX_DEPLOY_KEY` in its environment, and the admin access removes every form of it where a response is first seen; the record names the deployment as the backend it ran on, claims no parity under Sc L2-9 and passes no scenario; the hosted halves of Probe 3 and Probe 7, the production composition's acceptance on the deployment, and one local example that a table's documents survive another composition's deploy, are its examples | @@ -418,7 +418,7 @@ Things in the plan that the packages found wrong, and the convention adopted: Residual items, none of which fails validate or the rubric: -- Two environment variables gate a restore: `MAINTENANCE_MODE=restore` closes writers (transactional profile, E-42) and `OBLIGATIONS_DISPATCH=off` closes external dispatch (durable profile, E-56). Both are set before the snapshot is restored; `MAINTENANCE_MODE` is cleared first, in the transactional restore's accepted branch, then the schedule rebuild and reconcile pass run, and `OBLIGATIONS_DISPATCH` moves last. A single switch would be simpler and is the owner's call. +- Two environment variables gate a restore: `MAINTENANCE_MODE=restore` closes writers (transactional profile, E-42) and `OBLIGATIONS_DISPATCH=off` closes external dispatch (durable profile, E-56). Both are set before the backup archive is restored; `MAINTENANCE_MODE` is cleared first, in the transactional restore's accepted branch, then the schedule rebuild and reconcile pass run, and `OBLIGATIONS_DISPATCH` moves last. A single switch would be simpler and is the owner's call. - Packages B and C cite the Convex runtimes page by name inside E-26 and E-36 rather than by an S-number; the plan's Sources may want it. - Package E cites Workflow and Workpool README facts as S11 and S12 with the read date, beyond what the plan's section 9 records for those pages; the reviewer of rubric item 35 should confirm them against the READMEs. - The Design Review pages live under `generated/design-review/spec/` and `generated/design-review/pack/`, not one flat directory as the plan's section 0 implies. diff --git a/design/specs/application/first-experiment.order-of-1-line.sdp.md b/design/specs/application/first-experiment.order-of-1-line.sdp.md index c79ab51..92923d2 100644 --- a/design/specs/application/first-experiment.order-of-1-line.sdp.md +++ b/design/specs/application/first-experiment.order-of-1-line.sdp.md @@ -30,5 +30,5 @@ And the budgets {budgets: "hold"} - Runs in the native backend tier on the production composition; every test owns its disposable backend. - On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends `PlaceOrder` with one line, reads the backend's function log from a mark before the command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for the command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. -- The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation for this size. +- The call count is shown at the pure test tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation for this size. - The test reads the documents and bytes read and written from the completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts six documents written: the receipt, the order's state and event, the stock item's state and event, and the summary row; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-10-lines.sdp.md b/design/specs/application/first-experiment.order-of-10-lines.sdp.md index 0694cd7..c2e93a3 100644 --- a/design/specs/application/first-experiment.order-of-10-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-10-lines.sdp.md @@ -30,5 +30,5 @@ And the budgets {budgets: "hold"} - Runs in the native backend tier on the production composition; every test owns its disposable backend. - On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` with ten lines over ten stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row. -- The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation carrying the order and then one on the Inventory operation carrying all ten lines in one list. +- The call count is shown at the pure test tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation carrying the order and then one on the Inventory operation carrying all ten lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written, and asserts that the ten-line order wrote two documents more per added line than the one-line reference, the added stock item's state and its event, so documents written grew linearly with the lines; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. diff --git a/design/specs/application/first-experiment.order-of-max-lines.sdp.md b/design/specs/application/first-experiment.order-of-max-lines.sdp.md index bf2e288..fc111a6 100644 --- a/design/specs/application/first-experiment.order-of-max-lines.sdp.md +++ b/design/specs/application/first-experiment.order-of-max-lines.sdp.md @@ -30,7 +30,7 @@ And the budgets {budgets: "hold"} - Runs in the native backend tier on the production composition; every test owns its disposable backend. - On the production composition, after grants, the first rebuild of the order summary through to its switch and `ReceiveStock` as setup, the test sends a one-line order on another stock item as the reference and then `PlaceOrder` at the maximum of 100 lines over 100 stock items; for each it reads the backend's function log from a mark before that command to a read of the order summary list that the same client sends after the command returned, and asserts that the log holds one top-level mutation execution for that command, which committed, and no other execution, so no scheduled function, cron or action ran for it inside that window, which an ordinary read after the command closes, that `_scheduled_functions` of the parent and of both contexts, Orders and Inventory, read through admin access after the command, is empty, so no job is scheduled for later either, and that the list the read returned already holds the order's summary row; and it asserts that the execution time the completion record reports is under 1 second, a bound this example sets on that reported time and not the engine's cap on completion time, because the engine's limit is on the function's own computation and nested calls draw on a separate budget. -- The call count is shown at the pure tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation carrying all 100 lines in one list. +- The call count is shown at the pure test tier by the same executor, run on the order the test sent against a ctx whose `runMutation` counts calls by function reference and answers a canned operation outcome, which asserts one `runMutation` on the Orders operation and then one on the Inventory operation carrying all 100 lines in one list. - The test reads the documents and bytes read and written from each completion record's `usageStats`, records them on the test's entry in the run's evidence record and not as a `Measurement` record, and asserts them under the F13 ceilings of 32,000 documents and 16 MiB read and 16,000 documents and 16 MiB written; that the stream rows and the summary row stayed inside their budgets is shown by the command having committed, because a row above its budget fails the command. - The test then receives stock for 101 more stock items through `ReceiveStock`, sends `PlaceOrder` with one line on each, one line more than the maximum, and asserts the `operationTooLarge` rejection whose details carry 101 items against a maximum of 100, with the receipts, both contexts' stream rows and events and the order summaries unchanged. - The test records the cost per added line, the difference from the one-line reference divided by the 99 added lines. diff --git a/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md b/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md index 003c98a..e3a8f64 100644 --- a/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-placed-order.sdp.md @@ -29,6 +29,6 @@ And the number of documents the command wrote is {written: 6} - Runs in the native backend tier on the production composition; every test owns its disposable backend. - The order is `order-1` with two lines on the stock item `sku-1`, of 2 and 1 units; the test asserts that the stock item's state after the cancel equals its state before `PlaceOrder`, `{ onHand: 5, allocated: 0 }`. -- The test asserts the response `{ kind: "applied", replayed: false, result: { orderId: "order-1", released: [{ stockItemId: "sku-1", quantity: 3 }] } }` with versions naming the order at version 2 and the stock item at version 3, Orders first; the order of the two calls, Orders and then Inventory, is shown at the pure tier by the same executor run against a ctx that records each `runMutation` by its function reference, which this native test cannot tell apart. +- The test asserts the response `{ kind: "applied", replayed: false, result: { orderId: "order-1", released: [{ stockItemId: "sku-1", quantity: 3 }] } }` with versions naming the order at version 2 and the stock item at version 3, Orders first; the order of the two calls, Orders and then Inventory, is shown at the pure test tier by the same executor run against a ctx that records each `runMutation` by its function reference, which this native test cannot tell apart. - The six documents are the receipt, the order's state and its `OrderCancelled` event, the stock item's state and its one `AllocationReleased` event of quantity 3, and the summary row, whose `sourceVersions` name the order at version 2; the parent's `getOrder` answers the order with status `cancelled` and the lines, total and `placedAt` it answered before the cancel, and the summary row keeps that `placedAt`. - The test reads the function log and asserts that the command ran as one top-level mutation, and that the order summary list for `cancelled` holds the order and the list for `placed` does not. diff --git a/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md b/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md index 63a554c..e44ee47 100644 --- a/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md +++ b/design/specs/application/orders-inventory-example.cancel-second-context-rejects.sdp.md @@ -31,5 +31,5 @@ And the number of documents the command wrote is {written: 0} - Runs in the native backend tier on the production composition; every test owns its disposable backend. - No public command releases stock without cancelling its order, so the setup calls the Inventory context's release operation for the order's 3 units directly, with admin access, as a sanctioned operation of that context; the stock item's state stays the fold of its events, and the two contexts now disagree about the order. -- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "insufficientAllocation", entry: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 } }`; that the release's quantities come from the lines the Orders call returned, and that Orders is called first, is shown at the pure tier by the same executor run against a ctx that records each `runMutation` by its function reference. +- The test asserts that the caller's error is a `ConvexError` whose data is `{ kind: "rejection", code: "insufficientAllocation", entry: "CancelOrder", message: "Cannot release 3 when 0 are allocated", details: { requested: 3, allocated: 0 } }`; that the release's quantities come from the lines the Orders call returned, and that Orders is called first, is shown at the pure test tier by the same executor run against a ctx that records each `runMutation` by its function reference. - The test reads the function log and asserts that the request ran one top-level mutation, recorded as failed with no document written, and that the order's stream stays at version 1 with status `placed` and one event. diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 80f8dbd..ae1b70e 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -132,7 +132,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(Math.min(batchSize, limitPurgeBatch(readModel)))` with `batchSize` the progress row's and `readModel` the generation's read model, whose cap `limitPurgeBatch` declares, deletes each row and adds their number to `rowsPurged`; when it deleted that many rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`, a purge cursor being `{ tenantId, pageCursor: null }`; a pass that is not done checkpoints, patching the progress row with the cursor, `rowsPurged`, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) - purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) - fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator` and `updatedAt`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) -- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to no `lastError` and `updatedAt` and leaving its `batchSize`, cursor and counts as they are, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the snapshot stranded (D9, F8, E-40, E-42) +- fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to no `lastError` and `updatedAt` and leaving its `batchSize`, cursor and counts as they are, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the backup archive stranded (D9, F8, E-40, E-42) - fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain`; it takes no batch size, so a resumed pass runs at the progress row's `batchSize`, a purge capped by `limitPurgeBatch`; for a generation in any state whose progress row names a pass, so it continues a backfill or verify pass of a generation in `building` or `verifying` and a started purge of a `retired` or `aborted` one, which is how an operator continues a purge a restore stranded (D9, Sc L2-5, E-40, E-48) - fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) - fnRollbackGeneration: `export const rollbackGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` the operator names the retired generation to return to, and any retired generation inside its rollback period may be named; after `assertOperator` it throws `"Generation " + n + " of " + name + " is " + state + "; rollbackGeneration needs retired"` unless the state is `retired`, `"Generation " + n + " of " + name + " is past its rollback period"` when the mutation's `Date.now()` is at or after `retireAfter`, `"Generation " + n + " of " + name + " is already being purged"` when `pass` is `purge`, `"Generation " + n + " of " + name + " needs a write pause and cannot be rolled back"` when `pauseRequired` is set, `"The tenant list is being filled; rollbackGeneration waits until fillTenants has ended"` while the `tenantFill` row's `pass` is `fill`, the in-flight error of `startGeneration`, naming that generation's number and state, when another generation is `building`, `verifying` or `verified`, and the error of `projectionOf` when the read model no longer declares the generation's projection version; otherwise it patches the row to `state: "verifying"`, fence plus 1, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, patches the progress row to `pass: "verify"`, `cursor: null`, `misses: 0`, no `lastError` and `updatedAt`, with `batchSize` and the other counts kept, and schedules `refs.verifyBatch` with the new fence; the switch that completes the rollback is `switchGeneration` (D9, E-8, E-9, E-48) diff --git a/design/specs/application/restore.restore-representative-dataset.sdp.md b/design/specs/application/restore.restore-representative-dataset.sdp.md index caaa972..12db085 100644 --- a/design/specs/application/restore.restore-representative-dataset.sdp.md +++ b/design/specs/application/restore.restore-representative-dataset.sdp.md @@ -21,7 +21,7 @@ And every writer is refused while the restore door is closed When the restore procedure runs its journal, domain, read-model and receipt checks and its pending-work inventory Then the invariants {invariants: "hold"} And writes {reopen: "reopen after the checks pass"} -And a generation stranded in the snapshot is {stranded: "resumed by the accepted branch"} +And a generation stranded in the backup archive is {stranded: "resumed by the accepted branch"} ``` ## Verification — executable diff --git a/design/specs/application/restore.sdp.md b/design/specs/application/restore.sdp.md index 706ac36..2989bff 100644 --- a/design/specs/application/restore.sdp.md +++ b/design/specs/application/restore.sdp.md @@ -34,9 +34,9 @@ relations: Layer 2 · Detail: full · Traces: D3, D5, D6, D8, D19, Law 3, Law 8, Law 10, F8, F12, F13, S8, Probe 7, Sc L2-8, E-42, E-48. -Restore is the procedure that puts a backup's data under matching code and configuration and proves, before any writer runs, that the domain, journal and read-model invariants hold. A backup is a consistent snapshot of table data; it excludes code, configuration, environment variables and pending scheduled functions, and restoring it wipes the deployment's data and replaces it. The procedure therefore starts by closing the door to writers through the environment, deploys the matching commit, restores, runs its checks as bounded batches, takes an inventory of the pending work the snapshot carries but the scheduler no longer drives, reconciles the gap since the backup, and reopens only when every check passes. Because the snapshot holds the maintenance gate and the generation rows as they were at backup time but not the scheduled batch that was driving a rebuild, a generation left in `building` or `verifying` is stranded by the restore, and a closed gate entry that names it would keep its writers out for ever; the inventory lists every such row and the accepted branch resumes each generation the operator did not abort. Restore is tested, not only backup: the drill is the acceptance scenario, run on a representative dataset in a disposable deployment. +Restore is the procedure that puts a backup's data under matching code and configuration and proves, before any writer runs, that the domain, journal and read-model invariants hold. A backup archive holds the table data of one consistent moment; it excludes code, configuration, environment variables and pending scheduled functions, and restoring it wipes the deployment's data and replaces it. The procedure therefore starts by closing the door to writers through the environment, deploys the matching commit, restores, runs its checks as bounded batches, takes an inventory of the pending work the backup archive carries but the scheduler no longer drives, reconciles the gap since the backup, and reopens only when every check passes. Because the backup archive holds the maintenance gate and the generation rows as they were at backup time but not the scheduled batch that was driving a rebuild, a generation left in `building` or `verifying` is stranded by the restore, and a closed gate entry that names it would keep its writers out for ever; the inventory lists every such row and the accepted branch resumes each generation the operator did not abort. Restore is tested, not only backup: the drill is the acceptance scenario, run on a representative dataset in a disposable deployment. -The parts of restore that concern obligations, external dispatch and provider evidence belong to Layer 3 and to Package E. This workflow is the transactional profile's restore, which the durable profile extends: `spec:obligations.retention-and-restore` sets `OBLIGATIONS_DISPATCH` off before the snapshot is restored and runs its schedule rebuild and reconcile pass only after this workflow's accepted branch has cleared `MAINTENANCE_MODE`, so no rearmed wrapper runs into the closed door. +The parts of restore that concern obligations, external dispatch and provider evidence belong to Layer 3 and to Package E. This workflow is the transactional profile's restore, which the durable profile extends: `spec:obligations.retention-and-restore` sets `OBLIGATIONS_DISPATCH` off before the backup archive is restored and runs its schedule rebuild and reconcile pass only after this workflow's accepted branch has cleared `MAINTENANCE_MODE`, so no rearmed wrapper runs into the closed door. ## Intent @@ -47,14 +47,15 @@ The parts of restore that concern obligations, external dispatch and provider ev - risk: The checks are batches over every stream and every read-model row, so a large dataset takes many batches; the drill sizes the representative dataset so the checks finish inside the release window (F13, D19) - risk: Whether a backup and a restore cover component tables is not stated on the backup page; the contexts' data lives in components, so the drill proves it and Probe 7 records it (F12, S8, Probe 7) - assumption: Backups exclude pending scheduled functions, environment variables, code and configuration, and a restore is a destructive replacement of table data (F12, S8) -- assumption: Environment variables survive a restore, because they are not part of the snapshot (F12, S8) +- assumption: Environment variables survive a restore, because they are not part of the backup archive (F12, S8) - assumption: Scheduling from a mutation commits with it, so the batches the accepted branch reschedules for stranded generations exist only if the run record's acceptance committed (F8) ### Open questions - [non-blocking] Probe 7 observes local CLI replacement restoring the parent, Orders and Inventory documents with IDs and creation times; fresh replacement leaves scheduler tables empty and replacement in place keeps existing schedules; the complete Layer 2 restore drill, hosted dashboard restore, Workpool and Workflow remain open (Probe 7, F12, D19) -- [non-blocking] Extension E-42: the doc says restore runs with matching code and configuration, that domain, journal and read-model invariants hold, and that restore is tested; it does not list the checks or the record, and it does not say what a restore does to a rebuild that was in flight at the backup. The option taken here: four checks, journal, domain, read model and receipt, each a bounded checkpointed batch, the stream check bounded through the `rebuild` query's `maxEvents` argument so a batch folds at most 500 events in all, each folded stream also reading its own stored baseline event, a fifth step that inventories the closed gate entries, the generations in `building`, `verifying` or `verified` and the `imports` rows in `running` that the snapshot restored, a `restoreRuns` row as the record, an accepted branch that resumes every stranded generation the operator did not abort, the `MAINTENANCE_MODE` environment variable as the door, a failed check keeping the whole deployment closed with no override, and the drill before every release with its cadence reconsidered on the first drill's measured check times; the owner confirmed the checks, the door and both policies (E-42, D19, F12) +- [non-blocking] Extension E-42: the doc says restore runs with matching code and configuration, that domain, journal and read-model invariants hold, and that restore is tested; it does not list the checks or the record, and it does not say what a restore does to a rebuild that was in flight at the backup. The option taken here: four checks, journal, domain, read model and receipt, each a bounded checkpointed batch, the stream check bounded through the `rebuild` query's `maxEvents` argument so a batch folds at most 500 events in all, each folded stream also reading its own stored baseline event, a fifth step that inventories the closed gate entries, the generations in `building`, `verifying` or `verified` and the `imports` rows in `running` that the backup archive restored, a `restoreRuns` row as the record, an accepted branch that resumes every stranded generation the operator did not abort, the `MAINTENANCE_MODE` environment variable as the door, a failed check keeping the whole deployment closed with no override, and the drill before every release with its cadence reconsidered on the first drill's measured check times; the owner confirmed the checks, the door and both policies (E-42, D19, F12) - [non-blocking] Extension E-42, the stream check's batch size: `limitCheckBatch` states the read of a batch that folds one stream, and each further stream the same batch folds reads its own stored baseline event of at most `budgetBytes + 4,096 + 61` bytes; the batch size that keeps a batch of several folded streams under the 16 MiB read ceiling is not pinned (E-42, F13) +- [non-blocking] Extension E-42, the enumeration of a stream awaiting a baseline: the journal check enumerates streams through `list`, and `list` refuses a row behind a registered step that reads history with the transient refusal `writePaused`, so the check cannot reach such a stream to inventory it as `awaitingBaseline`; that enumeration needs a read that does not refuse it, designed together with the entry of the baseline-migration sweep (E-42, E-25) ## Workflow @@ -64,15 +65,15 @@ The parts of restore that concern obligations, external dispatch and provider ev - rule: [extension] Restore starts with external dispatch off; in the transactional profile that is the writer door closed through the environment until the checks pass (E-42, D19) - rule: The checks never run a command, never repeat an external effect and never rewrite an event (Law 10, D5) - rule: Every check is a bounded, checkpointed batch below the transaction limits (D19, F13) -- rule: After a restore the receipts in the snapshot are the authority for what committed; anything committed after the backup time is outside the restored state, and the run record states the backup time so callers can re-send with their request keys (D19, D6) +- rule: After a restore the receipts in the backup archive are the authority for what committed; anything committed after the backup time is outside the restored state, and the run record states the backup time so callers can re-send with their request keys (D19, D6) - rule: [extension] A restore that fails a check keeps the whole deployment closed, with no override, and the run record names the failed check and its subject; nobody reopens a deployment that failed a check (E-42, Sc L2-8, D19) - rule: [extension] The four checks are journal, domain, read model and receipt, in that order, and each records what it examined and what it found on the run record (E-42, D19) -- rule: [extension] A restore drops every pending scheduled function with the snapshot, so a generation in `building` or `verifying` at backup time comes back with no batch driving it and a closed gate entry that names it stays closed; the fifth step inventories both, plus every `imports` row in `running`, on the run record, and the accepted branch resumes every inventoried generation the operator did not abort, so the gate never stays closed for a rebuild nobody is running (E-42, F12, D19) +- rule: [extension] A restore drops every pending scheduled function, which the backup archive does not hold, so a generation in `building` or `verifying` at backup time comes back with no batch driving it and a closed gate entry that names it stays closed; the fifth step inventories both, plus every `imports` row in `running`, on the run record, and the accepted branch resumes every inventoried generation the operator did not abort, so the gate never stays closed for a rebuild nobody is running (E-42, F12, D19) - rule: [extension] A baseline-migration chain in flight at the backup leaves no row, only the baseline events it had written; the journal check inventories each stream it reports `awaitingBaseline` as pending work and not a finding, and the operator restarts the chain without a cursor after the accepted branch, which is safe because the journal's `writeBaseline` skips every stream already at the target schema version (E-42, F12, E-25, D5) - The operator picks the backup and the recovery-plan entry recorded with it: commit, schema hash, configuration hash, credential locations and the location of external outcome evidence (D19, Sc L2-8) - The operator sets `MAINTENANCE_MODE` to `restore` on the target deployment, so the writer check refuses every write with `writePaused` from now on (E-42, D19) - The operator deploys the recorded commit and configuration and confirms that the deployed schema matches the backup's schema hash (D19, Sc L2-8) -- The operator restores the snapshot into the target with matching code; native CLI replacement preserves the observed parent, Orders and Inventory documents, including IDs and creation times, while keeping destination environment variables and any existing scheduler rows; the dashboard path must be proved separately (S8, F12, Probe 7) +- The operator restores the backup archive into the target with matching code; native CLI replacement preserves the observed parent, Orders and Inventory documents, including IDs and creation times, while keeping destination environment variables and any existing scheduler rows; the dashboard path must be proved separately (S8, F12, Probe 7) - The operator starts the run record with `startRestoreRun`, which stores the backup ID, the backup time, the commit, the hashes and the operator (E-42, D19) - The journal check runs in batches: for each stream of a `list` page taken with `includeDeleted`, the context's `rebuild` query, called with the events the batch may still fold as `maxEvents`, reports `version`, `savedVersion`, `fromBaseline` and `eventsApplied`, and the check asserts that `eventsApplied` equals `version` minus the baseline version and that `version` equals `savedVersion`, which shows the events since the latest baseline are contiguous and the stream row agrees; a `rebuild` answer refused with `tailTooLong` is recorded as a journal finding naming the stream and its tail length, never retried with a bigger batch, while one refused with `budgetExhausted` is not a finding but the point where this batch stops and the next one resumes; event identity is enforced by the append's indexed read and is not rescanned (E-42, E-24, E-25, D2, D5) - The domain check runs in the same batches from the same `rebuild` call: its `equal` flag shows that folding the events since the latest baseline through `evolve` gives the saved state at the stream version (E-42, D3, Law 3, Sc L0-2) @@ -89,10 +90,10 @@ The parts of restore that concern obligations, external dispatch and provider ev The restore itself is a platform operation outside any mutation. Everything the procedure adds is in the parent: a run record table, one internal mutation per check batch, one for the pending-work inventory, and the door in the writer check. Each check batch is one mutation. The journal and domain checks share one batch function, because both read the context's `rebuild` query, which folds a stream inside the context and compares it with the saved row; that query is per stream and runs in the batch's own transaction, so the batch hands it the events it may still fold and the query refuses before reading when the tail would not fit, which makes one 500-event fold the batch's worst case rather than a fold on top of a nearly full budget. The read-model check reuses the rebuild's verify logic, and the receipt check reads the parent's receipts and the context's `byOperation` query. The pending-work inventory reads the gate document, the registry and the `imports` table and writes what it finds on the run record. The procedure's six mutations are the only mutations in the parent that do not read the write pause's gate: they write only the `restoreRuns` record and, in the accepted branch, the generation rows they resume; they run only while a run is `running`; and the door they hold closed would otherwise refuse them. -- transactionBoundary: none for the snapshot restore; each check batch is one internal mutation that reads through one component query and writes only the run record; `finishRestoreRun` is one mutation that marks the run and reschedules the stranded generations together, so acceptance and the rescheduling commit or fail as one (D19, F8, F13) +- transactionBoundary: none for the import of the backup archive; each check batch is one internal mutation that reads through one component query and writes only the run record; `finishRestoreRun` is one mutation that marks the run and reschedules the stranded generations together, so acceptance and the rescheduling commit or fail as one (D19, F8, F13) - convexSurface: `startRestoreRun`, `streamCheckBatch`, `readModelCheckBatch`, `receiptCheckBatch`, `pendingWorkBatch`, `finishRestoreRun` as internal mutations and `getRestoreRun` as an internal query, all in the parent (E-42, D19) - tableRestoreRuns: `restoreRuns: defineTable({ backupId: v.string(), backupTakenAt: v.number(), targetDeployment: v.string(), commit: v.string(), schemaHash: v.string(), configHash: v.string(), state: v.union(v.literal("running"), v.literal("accepted"), v.literal("failed")), checks: v.array(v.object({ name: v.string(), cursor: v.optional(v.string()), examined: v.number(), findings: v.array(v.object({ subject: v.string(), detail: v.string() })), state: v.union(v.literal("pending"), v.literal("running"), v.literal("passed"), v.literal("failed")) })), pendingWork: v.array(pendingWorkItemValidator), operator: v.string(), startedAt: v.number(), finishedAt: v.optional(v.number()), finishedBy: v.optional(v.string()) })` where `checks` holds five entries, the four checks and the inventory under the name `pendingWork`, whose cursor and state are kept like a check's while its items go to `pendingWork` (E-42, D19) -- validatorPendingWorkItem: `const pendingWorkItemValidator = v.object({ kind: v.union(v.literal("generation"), v.literal("closedScope"), v.literal("import")), subject: v.string(), detail: v.string(), generationId: v.optional(v.id("generations")), fence: v.optional(v.number()), resolution: v.optional(v.union(v.literal("resumed"), v.literal("aborted"), v.literal("kept"), v.literal("recorded"))) })` where `subject` is the read model and generation number, the scope key, or the tenant and import ID, `fence` is the generation's fence at inventory time, and `resolution` is written by `finishRestoreRun` (E-42, D19) +- validatorPendingWorkItem: `const pendingWorkItemValidator = v.object({ kind: v.union(v.literal("generation"), v.literal("closedScope"), v.literal("import"), v.literal("awaitingBaseline")), subject: v.string(), detail: v.string(), generationId: v.optional(v.id("generations")), fence: v.optional(v.number()), resolution: v.optional(v.union(v.literal("resumed"), v.literal("aborted"), v.literal("kept"), v.literal("recorded"))) })` where `subject` is the read model and generation number, the scope key, the tenant and import ID, or, for a stream awaiting a baseline, the context, stream type, tenant and stream ID, `fence` is the generation's fence at inventory time, and `resolution` is written by `finishRestoreRun`, an `awaitingBaseline` item's being `resumed` when the operator restarts the chain and `recorded` otherwise (E-42, E-25, D19) - validatorCheckScope: `const checkScopeValidator = v.union(v.object({ kind: v.literal("tenant"), tenantId: v.string() }), v.object({ kind: v.literal("all") }))` naming which tenants a check covers; under `all` a check batch iterates the tenant list through `nextTenant`, one tenant at a time, as the rebuild's `step2` and `step6` do, one tenant named on every context call, and never calls a context without a tenant (E-42, D11, Law 11) - indexRestoreRunsByStarted: `.index("by_started", ["startedAt"])` (E-42) - indexRestoreRunsByStartedUse: the operator's listing of runs, newest first (E-42, D19) @@ -119,7 +120,7 @@ And every writer is refused while the restore door is closed When the restore procedure runs its journal, domain, read-model and receipt checks and its pending-work inventory Then the invariants {invariants:"hold"|"fail"} And writes {reopen:"reopen after the checks pass"|"stay refused and the run record names the failed check"} -And a generation stranded in the snapshot is {stranded:"resumed by the accepted branch"|"left as it was"} +And a generation stranded in the backup archive is {stranded:"resumed by the accepted branch"|"left as it was"} ``` ## Verification — reviewed diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index 4cf706c..c4ba4da 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -20,7 +20,7 @@ relations: Story · Detail: full · Traces: D4, D9, D19, Law 6, F1, F9, F12, F13, Sc L2-6, Sc L2-8, E-8, E-42, E-43, E-48. -The write pause is the maintenance gate every writer in a scope reads inside its own mutation. It exists for the rebuild of a read model that depends on history across several streams, where a consistent cut is needed, and the restore procedure holds the same door closed for every scope through the environment, because the snapshot replaces the gate document, until its checks pass. A closed gate turns every write in scope into a transient refusal, retryable and never stored; a background batch scheduled before the close reads the closed gate when it starts, and a batch of a superseded chain is refused by its generation's fence. The rebuild closes the gate for its own scope in the mutation that registers its generation, an operator entry closes a tenant or the whole deployment, and operator entries resume and abort. Nothing reopens the gate by itself. Optimistic concurrency makes the close a consistent cut, so the gate has one closed state and no drain. +The write pause is the maintenance gate every writer in a scope reads inside its own mutation. It exists for the rebuild of a read model that depends on history across several streams, where a consistent cut is needed, and the restore procedure holds the same door closed for every scope through the environment, because the backup archive replaces the gate document, until its checks pass. A closed gate turns every write in scope into a transient refusal, retryable and never stored; a background batch scheduled before the close reads the closed gate when it starts, and a batch of a superseded chain is refused by its generation's fence. The rebuild closes the gate for its own scope in the mutation that registers its generation, an operator entry closes a tenant or the whole deployment, and operator entries resume and abort. Nothing reopens the gate by itself. Optimistic concurrency makes the close a consistent cut, so the gate has one closed state and no drain. ## Intent @@ -51,8 +51,8 @@ The write pause is the maintenance gate every writer in a scope reads inside its - [extension] The operator entries are `closeGate` and `resumeGate` for the scope `all` and for a tenant scope, and `getGate` and `getGateAudit` to read; each follows the operator entry contract of `spec:command.actor-and-scope`, and a source scope is closed and reopened only by the rebuild (E-8, E-48, D19) - Abort marks the generation aborted and resumes its scopes in one mutation (D9, Sc L2-6) - [extension] A source scope closes only for a generation `startGeneration` registers: `rollbackGeneration` of `spec:application.rebuild` refuses a retired generation with `pauseRequired`, because no function would close its sources while live commands fold onto rows that missed the events since its retirement, so a history view returns to earlier rules only through a new generation, whose start closes its sources again (E-8, D9, Sc L2-6) -- The restore procedure cannot use the gate document, because the snapshot replaces it; the restore door keeps every writer out through the environment, which survives a restore because backups exclude environment variables; the restore procedure's own mutations, which write only `restoreRuns` and the generation rows the accepted branch resumes with their progress rows, are the one exemption, and that workflow states it (D19, F12, Sc L2-8, E-42) -- [extension] A closed entry the snapshot brings back is as closed as one written live: an entry that names a generation stays closed while the restore's pending-work step resumes that generation, and reopens only when that generation is switched or an operator aborts it, and a tenant or `all` entry stays closed until an operator resumes it, so the gate never reopens by itself through a restore either (E-8, E-42, F12, D19) +- The restore procedure cannot use the gate document, because the backup archive replaces it; the restore door keeps every writer out through the environment, which survives a restore because backups exclude environment variables; the restore procedure's own mutations, which write only `restoreRuns` and the generation rows the accepted branch resumes with their progress rows, are the one exemption, and that workflow states it (D19, F12, Sc L2-8, E-42) +- [extension] A closed entry the backup archive brings back is as closed as one written live: an entry that names a generation stays closed while the restore's pending-work step resumes that generation, and reopens only when that generation is switched or an operator aborts it, and a tenant or `all` entry stays closed until an operator resumes it, so the gate never reopens by itself through a restore either (E-8, E-42, F12, D19) - [extension] Gate changes are treated as mandatory audit: every close and every resume inserts one record through `writeOperatorAudit` of `spec:operations.baseline-operations` inside the same mutation and fails closed, so a gate change whose audit insert fails changes nothing: the gate document is as it was and no record exists; the doc fails closed only for mandatory business or security audit and does not name gate changes among them (E-8, E-43, D19) - A local reaction refused by the gate is delayed, not counted: the obligation wrapper reschedules the same attempt without incrementing its attempt count, as `spec:obligations.local-reaction-wrapper` rules, so a pause never turns pending work into needs attention (D13, D9, E-51) @@ -67,7 +67,7 @@ The gate is one document so that a command pays one point read. Scopes are strin - fnGateAllows: `gateAllows(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[], allowGeneration?: Id<"generations">): Promise<{ allowed: true } | { allowed: false; scopeKey: ScopeKey; reason: string }>` the one gate read: it first reads `process.env.MAINTENANCE_MODE` and answers `{ allowed: false, scopeKey: "all", reason: "restore" }` without reading the document when the value is `restore`; otherwise it reads the gate document once and answers not allowed with the `scopeKey` and `reason` of the first entry of `closed`, in array order, whose `scopeKey` is one of `scopes` and which is not exempt, and allowed when there is none; an entry is exempt only when `allowGeneration` is given, the entry has a `generationId`, and the two are equal; it writes nothing (E-8, E-42, D9, F9) - fnAssertWritable: `assertWritable(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[]): Promise` calling `gateAllows` with no generation and, when the answer is not allowed, throwing through the boundary's `refuseTransient({ code: "writePaused", message: "write paused for " + scopeKey + ": " + reason })`; the command pipeline's step 7 calls it before it mints the operation, and `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` call it with `scopesOfUseCase(grant.tenantId, [])` before their first read; nothing else does (E-8, D9, E-35, E-37) - errorCodeWritePaused: `writePaused`, the transient code the outcome boundary reserves; the refusal's data is exactly `{ kind: "transient", code: "writePaused", message }` with no `retryAfterMs`, where `message` is `"write paused for all: restore"` under the restore door and `"write paused for tenant:" + tenantId + ": " + reason` for a closed tenant; a transient refusal, never a stored rejection, so no receipt, event, read-model row or audit record exists for the refused command (D4, Law 6, E-35) -- restoreDoor: `gateAllows` runs in the parent and reads `process.env.MAINTENANCE_MODE` before the gate document; only the exact value `restore` closes the door, so the restore procedure keeps writers out even though the snapshot replaced the gate, and every rebuild batch parks until the accepted branch resumes it; the restore procedure's own six mutations call neither `gateAllows` nor `assertWritable`; no component reads the variable (E-42, D19, F11) +- restoreDoor: `gateAllows` runs in the parent and reads `process.env.MAINTENANCE_MODE` before the gate document; only the exact value `restore` closes the door, so the restore procedure keeps writers out even though the backup archive replaced the gate, and every rebuild batch parks until the accepted branch resumes it; the restore procedure's own six mutations call neither `gateAllows` nor `assertWritable`; no component reads the variable (E-42, D19, F11) - fnScopesOfUseCase: `scopesOfUseCase(tenantId: string, writes: readonly SourceRef[]): ScopeKey[]` returning `"all"`, the tenant scope and one source scope per `SourceRef` of the declaration's `writes`, in that order, so a command is held by a closed deployment, by its closed tenant and by a paused rebuild of any view whose sources it writes (E-8, E-38, D9) - fnCloseScope: `closeScope(ctx: MutationCtx, scopeKey: ScopeKey, reason: string, operator: string, generationId?: Id<"generations">): Promise` throwing `"The scope " + scopeKey + " is already closed"` when `closed` holds an entry with that `scopeKey` and `"The gate holds 16 closed scopes, its limit"` when it holds `limitClosedScopes` entries; otherwise appending the entry to `closed`, inserting the document when there is none, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { kind: "gate.close", scopeKey, reason, generationId, operator })` in the caller's mutation; called by `closeGate` and by the rebuild's `startGeneration`, which passes the ID of the generation row it has just inserted (E-8, E-43, E-48, D9, D19) - fnResumeScope: `resumeScope(ctx: MutationCtx, scopeKey: ScopeKey, operator: string): Promise` throwing `"The scope " + scopeKey + " is not closed"` when `closed` holds no entry with that `scopeKey`; otherwise removing the entry, setting `updatedAt`, and calling `writeOperatorAudit(ctx, { kind: "gate.resume", scopeKey, reason: entry.reason, generationId: entry.generationId, operator })` in the caller's mutation, the reason and the generation being those of the entry it removed; called by `resumeGate` and by the rebuild's `abortGeneration` and `switchGeneration` (E-8, E-43, E-48, D9, D19) @@ -88,4 +88,4 @@ The gate is one document so that a command pays one point read. Scopes are strin - A reviewer confirms that no function other than `startGeneration` closes a source scope, so every closed entry that names a generation names a row that exists, and that the journal's baseline driver reads the gate with the source scope of the stream type it migrates before every batch. - A reviewer confirms that `closeScope` and `resumeScope` are the only writers of the gate document, that each inserts its audit record in the same mutation with no catch around it, and that a test whose audit table refuses the record finds the gate unchanged. - A reviewer confirms that `writePaused` appears in the outcome boundary's closed code list as a transient refusal and is never stored as a rejection. -- A reviewer confirms that the restore workflow sets `MAINTENANCE_MODE` before the snapshot is restored and clears it only after its checks. +- A reviewer confirms that the restore workflow sets `MAINTENANCE_MODE` before the backup archive is restored and clears it only after its checks. diff --git a/design/specs/command/command-declaration.sdp.md b/design/specs/command/command-declaration.sdp.md index 461a160..3f19d1f 100644 --- a/design/specs/command/command-declaration.sdp.md +++ b/design/specs/command/command-declaration.sdp.md @@ -63,7 +63,7 @@ What generation would emit, once two modules show the pattern: for each declarat - typePermissionPolicy: `type PermissionPolicy = { permission: string; subjectFrom?: (input: I) => SubjectRef }` (D11, D12) - typeExecutor: `type Executor = (ctx: MutationCtx, call: { tenantId: string; actor: Actor; operation: OperationRef; input: I }) => Promise>` where `OperationRef` is the one the pipeline minted at step 7, forwarded unchanged to every context call, and the executor's context calls are `ctx.runMutation` on component APIs with the facts it captured (D12, D10, D4, D3, E-26) - typeExecutorResult: `type ExecutorResult = CommittedOutcome & { streams: StreamDto[] }` where `CommittedOutcome` is the kernel's committed half of `Outcome` and `streams` concatenates the `streams` of every context outcome the executor received, the `StreamDto` of `spec:context.persistence-adapter`, so the pipeline's step 9 has every DTO, version and created flag it needs (D12, D4, D8, E-7, E-23) -- typeReadModelBinding: `type ReadModelBinding = { readModel: ReadModel | HistoryProjection; source: SourceRef }`, where `readModel` is a per-entity read model or a history projection as `spec:application.projection-contract` pins them and `source` selects which `streams` entries feed it; the binding repeats neither the read model's name nor its projection's version, a history projection is bound once per source stream type it folds while a per-entity read model is bound to its one source stream type, and the binding has no update function of its own (D8, D12, E-7, E-9) +- typeReadModelBinding: `type ReadModelBinding = { readModel: ReadModel | AggregateProjection | HistoryProjection; source: SourceRef }`, where `readModel` is a per-entity read model, an aggregate projection or a history projection as `spec:application.projection-contract` pins them and `source` selects which `streams` entries feed it; the binding repeats neither the read model's name nor its projection's version, a history projection is bound once per source stream type it folds while a per-entity read model or an aggregate projection is bound to its one source stream type, and the binding has no update function of its own (D8, D12, E-7, E-9) - typeAudit: `audit?: { kind: "security" | "business" }` asks the pipeline's step 10 to write one audit record through `writeAudit` of `spec:operations.baseline-operations`, which fails closed, after an applied command and after a business failure; absent, the default, the command writes none (D19, E-38, E-43) - typeDiagnosticSinkField: `diagnosticSink?: DiagnosticSink`, the sink type of `spec:operations.baseline-operations`, names where the pipeline writes the command's diagnostic lines; absent, the default, it is `consoleSink`; the production composition names none, and the fixture composition's `PlaceOrder` names the sink through which Sc ALL-1 breaks metrics and logging (D19, E-38, E-43) - typeAdmissionPolicy: `type AdmissionPolicy = (ctx: MutationCtx, call: PipelineCall) => Promise<{ admitted: true } | { admitted: false; code: "rateLimited" | "capacity"; retryAfterMs?: number }>`; a consumption it writes commits only with the command, so it counts committed intent (D6, E-32) diff --git a/design/specs/command/outcome-boundary.sdp.md b/design/specs/command/outcome-boundary.sdp.md index f65a2dd..5ee3061 100644 --- a/design/specs/command/outcome-boundary.sdp.md +++ b/design/specs/command/outcome-boundary.sdp.md @@ -36,7 +36,7 @@ The closed code list has two halves. The platform codes below are closed here an ### Open questions - [non-blocking] Extension E-5: the doc fixes the four outcomes and that a rejection is a structured `ConvexError`, but not the data shape or the code list; the shape and the platform codes below are the design's, and the domain codes are closed per declaration (E-5, D4, D7) -- [non-blocking] Extension E-5: the doc says a query authorizes before it discloses and does not say how a refused query is answered; here a parent query's refusal is the same `RejectionData` a command's is, with the name of the entry that refused in the field `entry` and only the codes `unauthenticated` and `forbidden`, and a use case's own rejection is thrown as the kernel's bare `Rejection`, the way a context throws one; the owner confirmed the shared shape, the two codes, the bare use-case rejection and the word entry for the field (E-5, D7, Law 5) +- [non-blocking] Extension E-5: the doc says a query authorizes before it discloses and does not say how a refused query is answered; here a parent query's refusal is the same `RejectionData` a command's is, with the name of the entry that refused in the field `entry` and only the codes `unauthenticated` and `forbidden`, and a use case's own rejection is thrown as the kernel's bare `Rejection`, the way a context throws one, while a context query's transient refusal `writePaused` reaches the caller unchanged, in the transient shape with no `entry`, because it is not a refusal of the caller; the owner confirmed the shared shape, the two codes, the bare use-case rejection and the word entry for the field (E-5, D7, Law 5) - [non-blocking] Extension E-35: the write pause of D9 refuses writers in its scope while a cross-stream history view rebuilds; the doc does not say how a refused writer is answered; this contract reserves the transient code `writePaused` for Package D so the refusal is retryable and stores nothing, which Package D confirms or replaces (E-35, D9, D4) - [non-blocking] OQ2: whether the product needs a record of refused commands anywhere, for security audit or agent proposals, decides whether the generic internal dispatcher below exists; until then it is a conditional design and no command uses it (OQ2, D7) @@ -44,7 +44,7 @@ The closed code list has two halves. The platform codes below are closed here an - Applied and business failure are returned as a `CommandResponse` value whose `kind` is `"applied"` or `"businessFailure"`; both commit, because a business failure is something meaningful that belongs in history (D4) - A rejection is thrown as `new ConvexError(data)` with `data.kind` equal to `"rejection"` and a code from the closed list; the throw prevents the mutation from committing and nothing is stored (D7, D4) -- A parent query's refusal is the same rejection: `new ConvexError(data)` with `data.kind` equal to `"rejection"`, a platform code, and the query's name in `entry`; a query throws no domain code, stores nothing and writes no receipt, and a client classifies it with the same test as a command's rejection (D7, Law 5) +- A parent query's refusal is the same rejection: `new ConvexError(data)` with `data.kind` equal to `"rejection"`, a platform code, and the query's name in `entry`; a query throws no domain code, stores nothing and writes no receipt, and a client classifies it with the same test as a command's rejection; a parent query relays the transient refusal `writePaused` of a context query unchanged, which is the transient shape of `errorDataValidator`, carries no `entry` and is not a refusal of the caller (D7, Law 5, E-35) - A technical failure is any throw that is not a `ConvexError` of this contract, such as a plain `Error`, a validator failure inside a component or an unexpected exception; everything rolls back, nothing is stored as a business outcome, and the boundary never converts it into a rejection (D4, Law 6) - A transient refusal is thrown as `new ConvexError(data)` with `data.kind` equal to `"transient"`; the caller may retry the same intent with the same request key, nothing is stored, and a valid retry can succeed (D4, D6, Sc L1-9) - The `data` of every `ConvexError` this contract throws is a value the `v` validators accept, so `errorDataValidator` below describes it exactly and a client reads it as `error.data` after `error instanceof ConvexError` (D7) @@ -68,7 +68,7 @@ The conditional dispatcher is the one place on the parent side where D7's nested - transactionBoundary: the pipeline's one top-level mutation; the conditional dispatcher adds one nested `ctx.runMutation` around the body for the commands that need a refusal record (D7, F3) - convexSurface: no registered function of its own; the error classes and validators are a library the pipeline imports; the dispatcher, if OQ2 says yes, is one `internalMutation` (D7, OQ2) - typeOutcomeMapping: `Outcome` of `spec:kernel.outcome-model` maps as applied → `CommandResponse` kind `"applied"`, businessFailure → `CommandResponse` kind `"businessFailure"`, rejection → thrown `ConvexError`; technical failure is any other throw (D4, D7) -- typeRejectionData: `type RejectionData = Rejection & { kind: "rejection"; code: RejectionCode; entry: string }` where `Rejection` is the kernel's `{ code: string; message: string; details?: Record }` from `spec:kernel.outcome-model`, never redefined here, and `entry` is the name of the public command or parent query that refused (D7, E-5) +- typeRejectionData: `type RejectionData = Rejection & { kind: "rejection"; code: RejectionCode; entry: string }` where `Rejection` is the kernel's `{ code: string; message: string; details?: Record }` from `spec:kernel.outcome-model`, never redefined here, and `entry` is the name of the entry that refused: a public command, an internal command, a parent query or an operator entry (D7, E-5) - typeTransientData: `type TransientData = { kind: "transient"; code: TransientCode; message: string; retryAfterMs?: number }` (D4, E-5) - typeRejectionCode: `type RejectionCode = PlatformRejectionCode | Domain` where `Domain` is the union of every code the command's declaration names in `rejections`, open as `string` by default (D4, D12, E-5) - typePlatformRejectionCode: `type PlatformRejectionCode = "invalidInput" | "unauthenticated" | "forbidden" | "idempotencyConflict" | "staleVersion" | "entityExists" | "operationTooLarge" | "unsupportedContractVersion"` (E-5) diff --git a/design/specs/context/context-component.sdp.md b/design/specs/context/context-component.sdp.md index ec9f25c..46dd4ea 100644 --- a/design/specs/context/context-component.sdp.md +++ b/design/specs/context/context-component.sdp.md @@ -38,7 +38,7 @@ The component's boundary is the component API. Below it sit the kernel, the jour - problem: A journal any parent code can write lets state change without its events; a central store puts a counter in every write's path; a context that reads `ctx.auth` decides authority twice; and without a boundary a stale reviewed version or a competing command is silently reinterpreted against fresh state (D2, D11, Sc L1-1, Sc L1-10, Sc L1-11) - outcome: Each context is a component that owns its CMS, journal and stream metadata behind sanctioned operations; the parent calls it through the component API with tenant, actor and operation as arguments, and every call commits or rolls back with the caller (D2, D11, F2) - value: Isolation is the platform's defining property; a context's private schema stays private, a cross-context reference is an application identity, and the rule that state never changes without its events is enforced where trusted code cannot break it by mistake (D2) -- risk: Every read of context data from the parent is a component call, which `spec:facts.f19-nested-calls-draw-on-a-system-operation-budget` reads on the fixture composition as 3.17 ms over a helper call on a quiet local backend, with every cost and speed target of the first experiment held; its cost in function-call quota is a hosted usage reading, and a measured workload that misses an agreed target, or a usage reading above an agreed share, reopens where hot reads land (F4, F19, Probe 3, OQ1) +- risk: Every read of context data from the parent is a component call, which `spec:facts.f19-nested-calls-share-time-budgets` reads on the fixture composition as 3.17 ms over a helper call on a quiet local backend, with every cost and speed target of the first experiment held; its cost in function-call quota is a hosted usage reading, and a measured workload that misses an agreed target, or a usage reading above an agreed share, reopens where hot reads land (F4, F19, Probe 3, OQ1) - risk: The standing cost per context is one component with its own tables, one registered mutation per sanctioned operation, up to four queries per stream type, one per definer it calls, one operation query, one maintenance mutation and a dependency on `convex-helpers` for pagination inside the component (D2, Decision method rule 2, E-22, E-24) - assumption: Component calls commit or roll back with the calling mutation, and each is a sub-transaction isolated from other calls (F2, S2) - assumption: The documentation says a component has no `ctx.auth`; the design does not rest on it, because by its own rule no component function reads `ctx.auth` and the parent passes actor and scope (F11, D11) diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md index 90af63c..f81a399 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.sdp.md @@ -17,7 +17,7 @@ Probe 7 · native backend tier · temporary copy of the production composition. ```gwt Given a production composition can store parent and context documents and typed scheduler references -When it creates documents and five scheduler states, then exports and imports the snapshot with replacement into a fresh backend with its own environment +When it creates documents and five scheduler states, then exports and imports the backup archive with replacement into a fresh backend with its own environment Then documents including ids and creation times in {scopes: "parent,orders,inventory"} equal the exported documents {equal: true} And the scheduler row counts are parent {parent: 0}, Orders {orders: 0} and Inventory {inventory: 0} And the destination environment is unchanged {unchanged: true} @@ -27,4 +27,4 @@ And each stored id validates as an argument and on a later write {valid: true}, ## Verification — executable - On the first run of this example, on 2026-10-02 on release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, the bound values held: the documents came back in the parent, Orders and Inventory, the scheduler tables stayed empty, and the stored scheduler ids still validated. -- The exported snapshot is the archive. Full document equality includes IDs and creation times. Hosted dashboard restore, file storage, Workpool and Workflow are not run. +- The exported file is the backup archive. Full document equality includes IDs and creation times. Hosted dashboard restore, file storage, Workpool and Workflow are not run. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md index e8b1be4..84f6259 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md @@ -17,7 +17,7 @@ Probe 7 · native backend tier · temporary copy of the production composition. ```gwt Given a production composition has temporary functions for scheduler references and reactions -When it exports business documents and five scheduler states, adds orders and schedules, then imports the snapshot in place before pending reactions are due +When it exports business documents and five scheduler states, adds orders and schedules, then imports the backup archive in place before pending reactions are due Then documents including ids and creation times in {scopes: "parent,orders,inventory"} equal the exported documents {equal: true} And every scheduler row in each scope is unchanged {unchanged: true} And later business changes are gone {removed: true} and the destination environment is unchanged {environmentUnchanged: true} @@ -29,4 +29,4 @@ And each restored scheduler reference resolves to the kept row {found: true} and - On the first run of this example, on 2026-10-02 on release `precompiled-2026-09-28-5c7cb5b` with `convex` 1.46.0, the bound values held: the scheduler rows were unchanged in all three scopes and the kept pending reactions read the restored value at or after their time. - The test changes business data in the parent and both contexts before import and fails if it does not reach that boundary. It records execution times; it binds only that execution is no earlier than the due time. -- The exported snapshot is the archive. Hosted dashboard restore, file storage, Workpool and Workflow are not run. +- The exported file is the backup archive. Hosted dashboard restore, file storage, Workpool and Workflow are not run. diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md index 88fe113..ff517c9 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.sdp.md @@ -10,7 +10,7 @@ relations: F12 · Status: documented · Doc status: Documented, S8 · Decisions: D13, D19. -The backup page excludes scheduled functions, code, configuration and environment variables. On the pinned native backend, CLI snapshot export and replacement import carry the parent, Orders and Inventory documents, including their IDs and creation times, but no scheduler table. Import into a fresh backend leaves its scheduler tables empty; import in place preserves its scheduler rows, including rows created after export. The inference that every restore leaves no dispatches does not hold for this CLI import. The destination environment variables are unchanged. Hosted dashboard restore is not run, so the fact retains documented status for that wider claim. Here snapshot means the exported archive, not current state or a baseline event. +The backup page excludes scheduled functions, code, configuration and environment variables. On the pinned native backend, a backup archive that the CLI exports and imports with replacement carries the parent, Orders and Inventory documents, including their IDs and creation times, but no scheduler table. Import into a fresh backend leaves its scheduler tables empty; import in place preserves its scheduler rows, including rows created after export. The inference that every restore leaves no dispatches does not hold for this CLI import. The destination environment variables are unchanged. Hosted dashboard restore is not run, so the fact retains documented status for that wider claim. The backup archive here is the exported file, not current state or a baseline event. ## Intent diff --git a/design/specs/obligations/retention-and-restore.sdp.md b/design/specs/obligations/retention-and-restore.sdp.md index 323cc6b..194b773 100644 --- a/design/specs/obligations/retention-and-restore.sdp.md +++ b/design/specs/obligations/retention-and-restore.sdp.md @@ -22,7 +22,7 @@ relations: Layer 3 · Detail: full where D13 and D19 rule; numbers provisional until the first experiment · Traces: D13, D14, D19, Law 8, F12, F13, F16, Sc L3-7, Sc L3-8, E-42, E-55, E-56. -Retention and restore are the two operations that make the obligation table earn its cost over a plain scheduled function. Retention never erases unresolved work and keeps completed work past every horizon that could repeat it. Restore starts with external dispatch off, reconciles the gap between the backup and the provider, and rebuilds schedules from the obligations, because a backup excludes pending scheduled functions. Handler keys and versions stay logical so that a release can keep old work executable through a shim. The durable profile's restore extends the transactional profile's: `spec:application.restore` closes the writer door with `MAINTENANCE_MODE`, restores the snapshot, runs its checks and clears the door in its accepted branch; only then does this Spec's schedule rebuild run, because a rearmed wrapper that met the closed door would only be deferred, and the reconcile pass must see a deployment whose checks passed. +Retention and restore are the two operations that make the obligation table earn its cost over a plain scheduled function. Retention never erases unresolved work and keeps completed work past every horizon that could repeat it. Restore starts with external dispatch off, reconciles the gap between the backup and the provider, and rebuilds schedules from the obligations, because a backup excludes pending scheduled functions. Handler keys and versions stay logical so that a release can keep old work executable through a shim. The durable profile's restore extends the transactional profile's: `spec:application.restore` closes the writer door with `MAINTENANCE_MODE`, restores the backup archive, runs its checks and clears the door in its accepted branch; only then does this Spec's schedule rebuild run, because a rearmed wrapper that met the closed door would only be deferred, and the reconcile pass must see a deployment whose checks passed. ## Intent @@ -31,7 +31,7 @@ Retention and restore are the two operations that make the obligation table earn ### Open questions - [non-blocking] Extension E-55: the doc gives no retention horizon, compaction rule or batch bound; this design keeps a completed obligation for the longest repetition policy validity in the module and never less than 30 days, compacts `lastError.message` and `lateEvidence` after 7 days into a count, and deletes at most 500 rows per retention run (D19, F13, E-55) -- [non-blocking] Extension E-56: the doc says restore starts with dispatch off but not how the switch works; this design reads a deployment environment variable in the parent's claim mutation only, where a call-mode claim defers its attempt without counting it and a reconcile-mode claim proceeds, sets it before the restore, adds a cursor-driven `rebuildSchedules` operation for a backlog larger than the sweeper's batch, and orders the two switches: `OBLIGATIONS_DISPATCH` goes off before the snapshot and on last, and `rebuildSchedules` refuses to run while `MAINTENANCE_MODE` is `restore`, so it runs only after `spec:application.restore` has accepted the run and reopened writers (D19, F12, E-42, E-56) +- [non-blocking] Extension E-56: the doc says restore starts with dispatch off but not how the switch works; this design reads a deployment environment variable in the parent's claim mutation only, where a call-mode claim defers its attempt without counting it and a reconcile-mode claim proceeds, sets it before the restore, adds a cursor-driven `rebuildSchedules` operation for a backlog larger than the sweeper's batch, and orders the two switches: `OBLIGATIONS_DISPATCH` goes off before the backup archive is restored and on last, and `rebuildSchedules` refuses to run while `MAINTENANCE_MODE` is `restore`, so it runs only after `spec:application.restore` has accepted the run and reopened writers (D19, F12, E-42, E-56) - [non-blocking] Probe 7 pending: what a restore leaves of Workpool and Workflow state decides whether the rebuild covers only the module's own dispatches or also passed-ownership work (Probe 7, F12, D19) ## Rule diff --git a/design/specs/platform/acceptance-contract.sdp.md b/design/specs/platform/acceptance-contract.sdp.md index 23b49f7..53ac1ce 100644 --- a/design/specs/platform/acceptance-contract.sdp.md +++ b/design/specs/platform/acceptance-contract.sdp.md @@ -41,7 +41,7 @@ The doc's acceptance table is the contract. It lists forty-three scenario rows b - A link between source and test is not a passing run (Acceptance scenarios) - A run with adjusted configuration states how it differs from production (Acceptance scenarios) - Readiness is stated plainly as specified, implemented, tested under named conditions, or operationally accepted (Acceptance scenarios) -- The first experiment passes when every Layer 0, 1 and 2 scenario passes on a native backend (First experiment) +- The first experiment passes when every Layer 0, 1 and 2 scenario and Sc ALL-1 passes, each at the tier this Spec names for it (First experiment, Sc ALL-1) - The one all-layer scenario, breaking metrics and logging and separately breaking mandatory audit, applies to every installed layer (Sc ALL-1) - [extension] A compiled-tier scenario passes when a typed caller that states the change fails `tsc` before any deployment; the compiled tier, which the doc's four tiers lack, is named by Sc L2-4 alone and needs no backend (E-14, Sc L2-4) - [extension] Three statements about a scenario stay apart: it is bound when a verifier of its example resolves in the graph, it passed when a run's record holds a passing result of that verifier, and a set of scenarios is accepted only when the acceptance check finds every one of them passed in one record (E-17, Acceptance scenarios) diff --git a/design/specs/platform/transactional-domain-platform.sdp.md b/design/specs/platform/transactional-domain-platform.sdp.md index 2a76fb9..825c253 100644 --- a/design/specs/platform/transactional-domain-platform.sdp.md +++ b/design/specs/platform/transactional-domain-platform.sdp.md @@ -35,7 +35,7 @@ Citations in this corpus use the tokens the plan fixes: D for a decision, Law fo - outcome: Adding domain sophistication does not add infrastructure: one mutation per business operation, contexts that own their state and journal, and layers each usable before the next exists (Thesis) - value: A new feature needs only its domain input, events and decision, one command or use-case binding, and the read model it actually uses; a maintainer traces its success and failure without unrelated queue, agent or governance registries (Thesis) - risk: Every read of context data from the parent is a component call, whose cost the first experiment measured on the local backend and which keeps every context a component; a measured workload that misses an agreed target, or a hosted usage reading above an agreed share, changes the shape of the context component before Layer 3 is designed (D2, Probe 3, OQ1) -- risk: Nothing has run yet; the sources support the Convex facts, not the correctness or performance of the design (Sources) +- risk: The sources support the Convex facts, not the correctness or performance of the design, which only the acceptance scenarios, each passing at its tier, show (Sources) - assumption: Mutations are serializable under optimistic concurrency, and component calls commit or roll back with the calling mutation (F1, F2) - assumption: One ledger fact is still assumed, and three that the doc lists as assumed are rechecked or probed in the corpus; each Spec that rests on one names it and the probe plan names the probe (F14, F15, F16, F17) @@ -53,7 +53,7 @@ Citations in this corpus use the tokens the plan fixes: D for a decision, Law fo - rule: The design succeeds when adding domain sophistication does not add infrastructure (Thesis) - rule: The twelve laws are the review surface; the decisions give the detail (Laws) - rule: Every mechanism is a decision whose first option is the do-nothing option that relies on Convex as it is, and only a failing acceptance scenario justifies a mechanism (Decision method rule 2, Decision method rule 3) -- rule: Layers 0 to 2 get full detail now; Layer 3 is written from what the first experiment shows; Layers 4 to 6 stay as a trigger, a promise and their scenarios until something triggers them (Decision method rule 4) +- rule: Layers 0 to 2 get full detail; Layer 3 is written from what the first experiment shows; Layers 4 to 6 stay as a trigger, a promise and their scenarios until something triggers them (Decision method rule 4) - rule: The design authorizes no deletion or migration of existing data (Existing systems) ## Verification — reviewed diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts index b583259..a13685b 100644 --- a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import.test.ts @@ -27,7 +27,7 @@ bindExample(contract, (): World => ({}), { async (world) => { Object.assign(world, await schedulingBackend()); }, - "it creates documents and five scheduler states, then exports and imports the snapshot with replacement into a fresh backend with its own environment": + "it creates documents and five scheduler states, then exports and imports the backup archive with replacement into a fresh backend with its own environment": async (world) => { world.observation = await freshImport(world.backend!, world.directory!); }, diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts index 3de2a60..ae95328 100644 --- a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts @@ -27,7 +27,7 @@ bindExample(contract, (): World => ({}), { async (world) => { Object.assign(world, await schedulingBackend()); }, - "it exports business documents and five scheduler states, adds orders and schedules, then imports the snapshot in place before pending reactions are due": + "it exports business documents and five scheduler states, adds orders and schedules, then imports the backup archive in place before pending reactions are due": async (world) => { world.observation = await inPlaceImport(world.backend!, world.directory!); }, From 55a93c7c1ed5881135d0cfd7e9f6f2da1bb52e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:34:19 +0200 Subject: [PATCH 140/155] build: close fold-in F14 F15 F16 F17 F19 F22 The receipt sweep refuses a now later than the mutation's time, reads through boundedPage with list's byte bound so a run ends at 1,000 rows or 8 MiB, deletes what it read without the dead classification, and checks now and limit first in sweepNext too; both are exported through the command index, and the receipt Specs name fnSweepNext and the byte bound. convex-test cases show each refusal and the early end. Co-Authored-By: Claude Opus 5.5 --- .../command/idempotency-and-receipts.sdp.md | 4 +- design/specs/command/receipt-table.sdp.md | 3 +- example/convex/receipts.ts | 2 +- fixture/convex/receipts.ts | 2 +- src/command/index.ts | 9 ++ src/command/receipts.ts | 70 ++++++----- tests/simulator/receipts-sweep.test.ts | 111 ++++++++++++++++++ 7 files changed, 168 insertions(+), 33 deletions(-) diff --git a/design/specs/command/idempotency-and-receipts.sdp.md b/design/specs/command/idempotency-and-receipts.sdp.md index 014b8c4..9cfe2db 100644 --- a/design/specs/command/idempotency-and-receipts.sdp.md +++ b/design/specs/command/idempotency-and-receipts.sdp.md @@ -83,7 +83,7 @@ The receipt read and insert are helpers inside the pipeline's mutation, not regi Expiry is lazy on the core path and swept in the background. A lookup that finds an expired reversible receipt deletes it and proceeds as new intent; a lookup that finds an expired irreversible receipt compacts it to a tombstone and answers as a duplicate. The sweep is one bounded batch per tenant that removes expired reversible receipts and compacts expired irreversible ones; it is an operations duty and never decides an outcome. - transactionBoundary: read-check-insert inside the pipeline's one top-level mutation; the sweep is a separate bounded mutation per tenant (D6, D19) -- convexSurface: helpers `lookupReceipt`, `classifyReceipt`, `insertReceipt`, `fingerprintOf`; one `internalMutation` `receipts.sweep` per deployment, scheduled by operations, never on the command path (D6, E-34) +- convexSurface: helpers `lookupReceipt`, `classifyReceipt`, `insertReceipt`, `fingerprintOf`; two `internalMutation`s per deployment, `receipts.sweep`, one bounded run for one tenant, and `receipts.sweepNext`, one such run for the next tenant of the tenant list after the caller's cursor, as `fnSweep` and `fnSweepNext` of `spec:command.receipt-table` pin them; operations calls them, nothing schedules them, and neither is on the command path (D6, E-34) - typeReceiptKey: `type ReceiptKey = { tenantId: string; namespace: CallerNamespace; commandType: string; requestKey: string }` (D6, Law 4) - typeReceiptClass: `type ReceiptClass = { class: "new" } | { class: "duplicate"; receipt: Doc<"receipts"> } | { class: "tombstone"; receipt: Doc<"receipts"> } | { class: "conflict" } | { class: "unsupportedVersion" }`, where the conflict and the unsupported-version members carry no fields of the row, so no caller of `classifyReceipt` holds a field of the stored receipt to relay; the conflict and unsupported-version answers still disclose the one fact the two codes cannot hide, whether the receipt's contract version is the declaration's (D6, Law 5, E-5) - fnFingerprintOf: `fingerprintOf(input: unknown, contractVersion: number): Promise` returns the lowercase hex SHA-256 of `canonicalJson(input) + "\n" + contractVersion` (D6, E-36) @@ -99,7 +99,7 @@ Expiry is lazy on the core path and swept in the background. A lookup that finds - retentionDefault: `retention: { window: 7 * 24 * 60 * 60 * 1000, afterExpiry: "delete" | "tombstone" }` per declaration; `"tombstone"` is required when `irreversible` is true and the command has no domain uniqueness rule (D6, E-34) - limitIdLength: [extension] `limitIdLength` is 256 bytes of UTF-8, measured with `utf8Length` of `spec:command.command-pipeline`, for the tenant ID and the request key, a longer one is rejection `invalidInput` at step 1, and for the declaration's name, a longer one is a plain error; the count is of bytes, never of characters, so one number sizes the receipt row and the event envelope (E-34) - limitAffectedRefs: at most 1,000 affected refs and 1,000 stream versions per receipt, checked by `insertReceipt`; the row-size arithmetic against the 1 MiB document limit is `limitRowSize` in `spec:command.receipt-table`, and this Spec states no figure of its own (F13, E-34) -- limitSweepBatch: `receipts.sweep` handles at most 1,000 receipts per run for one tenant, reading `by_tenant_expiry` up to `now`, so one run stays far under the 16,000 documents written per transaction; the limit is a ceiling, not a target (D19, F13, E-34) +- limitSweepBatch: `receipts.sweep` handles at most 1,000 receipts per run for one tenant, reading `by_tenant_expiry` up to `now`, so one run stays far under the 16,000 documents written per transaction; its read is bounded by bytes as well, through `boundedPage` with `limitListBytes` of `spec:context.queries`, because 1,000 rows near the row bound of `limitRowSize` would pass the 16 MiB read limit, so a run ends at 1,000 rows or at 8 MiB read, whichever comes first, and answers `more`; the limit is a ceiling, not a target (D19, F13, E-34) - serializationArgument: both concurrent requests read the `by_key` range and find nothing; the first to commit inserts into that range; the second's commit is refused because its read set changed, a read set that holds the receipt range and the context's rows, so the conflict the engine reports may name either, the engine retries it, and the retry reads the receipt; the engine retry is invisible to the caller and is not a business attempt (F1, D15) - exampleActorKinds: the examples of this Spec's space run a caller in namespace `public` as a `human` actor, `worker` and `service` as a `service` actor and `agent` as an `agent` actor; the internal entry takes the actor and the namespace as two arguments, so the namespace alone fixes no actor kind (Sc L1-3, D11, E-6) diff --git a/design/specs/command/receipt-table.sdp.md b/design/specs/command/receipt-table.sdp.md index 3a923f9..34591b0 100644 --- a/design/specs/command/receipt-table.sdp.md +++ b/design/specs/command/receipt-table.sdp.md @@ -56,7 +56,8 @@ The `receipts` table lives in the parent, as the doc's ownership table assigns i - expiryDefaults: `expiresAt = recordedAt + retention.window` with a default window of 7 days; `tombstoneHorizon` defaults to `Number.MAX_SAFE_INTEGER`, meaning never, and a declaration may set a finite horizon (D6, E-4) - limitRowSize: [extension] at most 1,000 affected refs and 1,000 versions, the counts `insertReceipt` checks, keep a row under 400 KiB, below the 1 MiB document limit with room for the key fields, only while each entry measures under 200 bytes; the tenant ID, the request key and the command type are each at most `limitIdLength` bytes of UTF-8, checked at the pipeline's step 1; a stream ID inside an entry is at most 256 bytes, checked at the persistence adapter's step 1, and no check holds an entry under 200 bytes, so a row above the document limit fails at its insert as a technical failure; this is the one figure for receipt size in the corpus, and `spec:command.idempotency-and-receipts` cites it rather than restating it (F13, E-4) - limitIndexes: three indexes of at most four fields, within the 32 indexes and 16 fields per index the platform allows (F13) -- fnSweep: `export const sweep = internalMutation({ args: { tenantId: v.string(), now: v.number(), limit: v.number() }, returns: v.object({ deleted: v.number(), compacted: v.number(), more: v.boolean() }), handler })` with `limit` capped at 1,000 (D19, E-4) +- fnSweep: `export const sweep = internalMutation({ args: { tenantId: v.string(), now: v.number(), limit: v.number() }, returns: v.object({ deleted: v.number(), compacted: v.number(), more: v.boolean() }), handler })` with `limit` capped at 1,000; it refuses with a plain error a `now` that is not finite or is later than the mutation's `Date.now()` and a `limit` that is not a positive safe integer, and deletes nothing then; it reads expired rows through `boundedPage` of `spec:context.queries` with the byte bound `limitListBytes`, so a run ends at `min(limit, 1000)` rows or at that byte bound, whichever comes first, deletes each row it read and answers `more` when an expired row remains (D19, E-4, F13) +- fnSweepNext: `export const sweepNext = internalMutation({ args: { after: v.union(v.string(), v.null()), now: v.number(), limit: v.number() }, returns: v.object({ deleted: v.number(), compacted: v.number(), more: v.boolean(), tenantId: v.union(v.string(), v.null()), after: v.union(v.string(), v.null()) }), handler })` checks `now` and `limit` as `sweep` does before anything else, takes the first tenant after `after` from the tenant list through `nextTenant` and runs one `sweep` for it, one tenant per run; it answers that tenant with the sweep's counts and the cursor for the next run, which stays `after` while the tenant has `more` and becomes the tenant otherwise, and a `tenantId` of `null` with zero counts, `more` false and `after` null when no tenant follows, which ends the pass; the caller holds the cursor between runs, and nothing schedules a run (D19, E-4, Law 11) ## Verification — reviewed diff --git a/example/convex/receipts.ts b/example/convex/receipts.ts index 590aac1..ed8cdc2 100644 --- a/example/convex/receipts.ts +++ b/example/convex/receipts.ts @@ -1,5 +1,5 @@ // The composition's receipt sweep, called by operations. -import * as receipts from "../../src/command/receipts.js"; +import * as receipts from "../../src/command/index.js"; import { internalMutation } from "./_generated/server.js"; export const sweep = internalMutation({ args: receipts.sweepArgs, diff --git a/fixture/convex/receipts.ts b/fixture/convex/receipts.ts index 590aac1..ed8cdc2 100644 --- a/fixture/convex/receipts.ts +++ b/fixture/convex/receipts.ts @@ -1,5 +1,5 @@ // The composition's receipt sweep, called by operations. -import * as receipts from "../../src/command/receipts.js"; +import * as receipts from "../../src/command/index.js"; import { internalMutation } from "./_generated/server.js"; export const sweep = internalMutation({ args: receipts.sweepArgs, diff --git a/src/command/index.ts b/src/command/index.ts index a2b020c..635c6a8 100644 --- a/src/command/index.ts +++ b/src/command/index.ts @@ -74,13 +74,22 @@ export { fingerprintOf, insertReceipt, limitAffectedRefs, + limitSweepBatch, + limitSweepBytes, lookupReceipt, + sweep, + sweepArgs, + sweepNext, + sweepNextArgs, + sweepNextResultValidator, + sweepResultValidator, } from "./receipts.js"; export type { ReceiptClass, ReceiptInsert, ReceiptKey, Retention, + SweepResult, } from "./receipts.js"; export { commandTables } from "./tables.js"; export type { diff --git a/src/command/receipts.ts b/src/command/receipts.ts index 293b353..654f093 100644 --- a/src/command/receipts.ts +++ b/src/command/receipts.ts @@ -1,11 +1,14 @@ // Receipts of spec:command.idempotency-and-receipts: the key, the fingerprint, the lookup, the -// classification and the insert. Helpers inside the pipeline's mutation, never registered functions. -import { convexToJson, type JSONValue, type Value } from "convex/values"; +// classification and the insert, helpers inside the pipeline's mutation; and the sweep and its loop +// over tenants, handlers that each composition registers as internal mutations for operations. +import { paginator } from "convex-helpers/server/pagination"; +import { defineSchema } from "convex/server"; +import { convexToJson, v, type JSONValue, type Value } from "convex/values"; +import { boundedPage, limitListBytes } from "../context/queries.js"; import type { AffectedRef, StreamVersion } from "../kernel/index.js"; import type { CallerNamespace } from "./actor-and-scope.js"; -import type { MutationCtx, Receipt } from "./tables.js"; import { nextTenant } from "./authority.js"; -import { v } from "convex/values"; +import { commandTables, type MutationCtx, type Receipt } from "./tables.js"; export type ReceiptKey = { tenantId: string; namespace: CallerNamespace; @@ -117,8 +120,12 @@ export async function insertReceipt( }); } -// spec:command.receipt-table fnSweep. Operations supplies the time and batch size. +// spec:command.receipt-table fnSweep and fnSweepNext. Operations supplies the time and batch size. export const limitSweepBatch = 1000; +// The byte bound of a sweep's read: list's bound, through the same boundedPage helper. +export const limitSweepBytes = limitListBytes; +// paginator reads index fields from a schema: the command tables the parent's schema spreads. +const commandSchema = defineSchema(commandTables); export const sweepArgs = { tenantId: v.string(), now: v.number(), @@ -130,38 +137,44 @@ export const sweepResultValidator = v.object({ more: v.boolean(), }); export type SweepResult = { deleted: number; compacted: number; more: boolean }; -export async function sweep( - ctx: MutationCtx, - { tenantId, now, limit }: { tenantId: string; now: number; limit: number }, -): Promise { +// A bad now or limit is a plain error, and so is a now later than the mutation's Date.now(), which +// would delete receipts still inside their window. +function checkSweepArgs(now: number, limit: number) { if (!Number.isFinite(now)) throw new Error("Receipt sweep now must be finite"); + if (now > Date.now()) + throw new Error( + "Receipt sweep now must not be later than the current time", + ); if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("Receipt sweep limit must be a positive safe integer"); - const size = Math.min(limit, limitSweepBatch); +} +// One run reads expired rows until it holds min(limit, 1000) or has read limitSweepBytes, whichever +// comes first, and deletes each row it read, an expired tombstone included. Compaction waits for +// tombstones, so compacted is 0. +export async function sweep( + ctx: MutationCtx, + { tenantId, now, limit }: { tenantId: string; now: number; limit: number }, +): Promise { + checkSweepArgs(now, limit); const expired = () => - ctx.db + paginator(ctx.db, commandSchema) .query("receipts") .withIndex("by_tenant_expiry", (q) => q.eq("tenantId", tenantId).lte("expiresAt", now), ); - const rows = await expired().take(size); - let deleted = 0; - for (const row of rows) { - if (!row.tombstone) { - const classified = classifyReceipt( - row, - row.fingerprint, - row.contractVersion, - now, - ); - if (classified.class !== "new") - throw new Error("Receipt sweep expected an expired receipt"); - } - await ctx.db.delete(row._id); - deleted++; - } - return { deleted, compacted: 0, more: (await expired().first()) !== null }; + const { page } = await expired().paginate( + boundedPage( + { cursor: null, numItems: limit }, + { items: limitSweepBatch, bytes: limitSweepBytes }, + ), + ); + for (const row of page) await ctx.db.delete(row._id); + return { + deleted: page.length, + compacted: 0, + more: (await expired().first()) !== null, + }; } // The operations caller carries after between runs. A tenant with more stays on the same cursor; // a null tenant ends the pass. Nothing schedules another run. @@ -179,6 +192,7 @@ export async function sweepNext( ctx: MutationCtx, args: { after: string | null; now: number; limit: number }, ): Promise { + checkSweepArgs(args.now, args.limit); const tenantId = await nextTenant(ctx, args.after); if (tenantId === null) return { tenantId, after: null, deleted: 0, compacted: 0, more: false }; diff --git a/tests/simulator/receipts-sweep.test.ts b/tests/simulator/receipts-sweep.test.ts index 3aeb3b4..8c5fd77 100644 --- a/tests/simulator/receipts-sweep.test.ts +++ b/tests/simulator/receipts-sweep.test.ts @@ -1,5 +1,7 @@ import { convexTest, type TestConvex } from "convex-test"; import type { SchemaDefinition } from "convex/server"; +import { getDocumentSize } from "convex/values"; +import { limitSweepBytes } from "../../src/command/index.js"; import { commandTables } from "../../src/command/tables.js"; import { expect, test } from "vitest"; import schema from "../../fixture/convex/schema.js"; @@ -118,6 +120,81 @@ for (const composition of compositions) { }), ).toEqual({ deleted: 2, compacted: 0, more: true }); }); + // spec:command.receipt-table fnSweep: a bad now, a now later than the current time. + test(`convex-test: ${composition.name} sweep refuses a non-finite now and a now later than the current time, and deletes nothing`, async () => { + const t = composition.app(); + const later = Date.now() + 60 * 60 * 1000; + const stored = await t.run(async (ctx) => { + await ctx.db.insert("receipts", receipt("a", "expired")); + await ctx.db.insert("receipts", receipt("a", "live", later - 1)); + await ctx.db.insert("receipts", { + ...receipt("a", "tombstone", Number.MAX_SAFE_INTEGER), + tombstone: true, + affected: [], + versions: [], + }); + return ctx.db.query("receipts").collect(); + }); + for (const [bad, message] of [ + [NaN, "must be finite"], + [Infinity, "must be finite"], + [-Infinity, "must be finite"], + [later, "must not be later than the current time"], + ] as const) + await expect( + t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now: bad, + limit: 1000, + }), + ).rejects.toThrow(message); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual( + stored, + ); + }); + // spec:command.idempotency-and-receipts limitSweepBatch: the byte bound of a run. + test(`convex-test: ${composition.name} sweep ends a run at the byte bound and reports more`, async () => { + const t = composition.app(); + const rows = await t.run(async (ctx) => { + for (let i = 0; i < 20; i++) + await ctx.db.insert("receipts", { + ...receipt("a", `large-${i}`), + actorId: "x".repeat(512 * 1024), + }); + return ctx.db + .query("receipts") + .withIndex("by_tenant_expiry", (q) => q.eq("tenantId", "a")) + .collect(); + }); + // The rows read up to and including the one whose size reaches the bound. + let bytes = 0; + let read = 0; + for (const row of rows) { + bytes += getDocumentSize(row); + read++; + if (bytes >= limitSweepBytes) break; + } + expect(read).toBeLessThan(rows.length); + expect( + await t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 1000, + }), + ).toEqual({ deleted: read, compacted: 0, more: true }); + expect( + (await t.run((ctx) => ctx.db.query("receipts").collect())).map( + (row) => row.requestKey, + ), + ).toEqual(rows.slice(read).map((row) => row.requestKey)); + expect( + await t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 1000, + }), + ).toEqual({ deleted: rows.length - read, compacted: 0, more: false }); + }); // spec:command.receipt-table tombstoneShape, expiryDefaults and fnSweep. test(`convex-test: ${composition.name} deletes an expired tombstone and leaves an unexpired tombstone untouched`, async () => { const t = composition.app(); @@ -216,4 +293,38 @@ for (const composition of compositions) { ), ).toEqual([]); }); + // spec:command.receipt-table fnSweepNext checks now and limit before anything else. + test(`convex-test: ${composition.name} sweep loop refuses a bad now or limit whether or not a tenant follows`, async () => { + const t = composition.app(); + const later = Date.now() + 60 * 60 * 1000; + const bad = [ + [{ now: NaN, limit: 1 }, "must be finite"], + [{ now: Infinity, limit: 1 }, "must be finite"], + [{ now: later, limit: 1 }, "must not be later than the current time"], + [{ now, limit: 0 }, "positive safe integer"], + [{ now, limit: 1.5 }, "positive safe integer"], + ] as const; + for (const after of [null, "z"]) + for (const [args, message] of bad) + await expect( + t.mutation(composition.internal.receipts.sweepNext, { + after, + ...args, + }), + ).rejects.toThrow(message); + await t.run(async (ctx) => { + await ctx.db.insert("tenants", { tenantId: "a", createdAt: 0 }); + await ctx.db.insert("receipts", receipt("a", "expired")); + }); + for (const [args, message] of bad) + await expect( + t.mutation(composition.internal.receipts.sweepNext, { + after: null, + ...args, + }), + ).rejects.toThrow(message); + expect( + await t.run((ctx) => ctx.db.query("receipts").collect()), + ).toHaveLength(1); + }); } From 8d4f511d1608f444f00885ed60806f93e6a3c8c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:38:58 +0200 Subject: [PATCH 141/155] build: close fold-in F18 F20 F21 F23 F43 The versions cut of a diagnostic line finds its longest fitting prefix by a binary search, which a pure test at 1,000 versions holds equal to the linear search; a sink that returns a thenable has its record counted as lost by one gap line and its rejection handled, as fnEmitDiagnostic now says. The pure fit test fails a build that leaves versionsOmitted out, a convex-test case shows a skipped generation left out of readModelRows, and the support helper, its virtual modules, a title and a comment say what they are. Co-Authored-By: Claude Opus 5.5 --- .../operations/baseline-operations.sdp.md | 2 +- src/operations/diagnostic.ts | 28 +++++-- tests/pure/diagnostic.test.ts | 52 ++++++++++++- tests/simulator/diagnostic-support.ts | 6 +- tests/simulator/diagnostic.test.ts | 75 +++++++++++++++++-- 5 files changed, 150 insertions(+), 13 deletions(-) diff --git a/design/specs/operations/baseline-operations.sdp.md b/design/specs/operations/baseline-operations.sdp.md index 922c326..044d595 100644 --- a/design/specs/operations/baseline-operations.sdp.md +++ b/design/specs/operations/baseline-operations.sdp.md @@ -93,7 +93,7 @@ Optimistic-concurrency retries cannot be observed from inside a mutation, becaus - diagnosticSourcesThrown: [extension] for a command that throws, the entry's outermost catch takes `tenantId`, `requestKey`, `correlationId` and, when the caller states one, `causedBy` from the entry's arguments, `namespace` from the internal entry's argument or `public` for the public entry, and `actorKind` and `actorId` only when an actor exists, the internal entry's argument or the actor the public entry's step 2 established; `commandType` comes from the declaration's `name`; `kind` and `code` come from `classifyThrown` of what the catch throws, `rejection` or `transient` with the error data's `code`, or `technical` with no `code`; `replayed` is false; `operationId` is absent, because nothing stored carries it; `eventsAppended` and `readModelRows` are 0 and `versions` is empty, because nothing committed (E-43, D4, D7, D19) - fnDiagnosticLine: [extension] `diagnosticLine(record: DiagnosticRecord): string` returns `diagnostic ` followed by `JSON.stringify` of an object holding the record's fields in the order of `typeDiagnosticRecord`, whatever the order of the record's own keys, with absent optional fields left out and no other key; when that line measures more than `limitDiagnosticBytes` with `utf8Length`, `versions` is cut to its longest prefix for which the line fits, with the key `versionsOmitted`, the number of entries cut, placed just before `versions` and counted in the fit; when the line does not fit even with no entry kept, it is cut to its longest prefix of whole code points that measures at most `limitDiagnosticBytes` (E-43, D19) - fnDiagnosticGapLine: [extension] `diagnosticGapLine(record: DiagnosticRecord): string` returns `diagnostic gap ` followed by `JSON.stringify` of `{ tenantId, commandType, kind }` from the record, with `operationId` after them when the record has one, cut as `diagnosticLine` cuts a line that does not fit (E-43, D19) -- fnEmitDiagnostic: [extension] `emitDiagnostic(record: DiagnosticRecord, sink: DiagnosticSink = consoleSink): void` calls `sink(diagnosticLine(record))` once inside a try; when that throws, whatever is thrown, it calls `console.error(diagnosticGapLine(record))` once inside a second try whose catch does nothing; it never throws, reads and writes no table, and itself writes no line but the gap line (E-43, D19, Sc ALL-1) +- fnEmitDiagnostic: [extension] `emitDiagnostic(record: DiagnosticRecord, sink: DiagnosticSink = consoleSink): void` calls `sink(diagnosticLine(record))` once inside a try; a sink is synchronous, so when it returns a thenable, `emitDiagnostic` gives that thenable a rejection handler that does nothing, so no rejection escapes, and counts the record as lost as if the sink had thrown; when that throws, whatever is thrown, it calls `console.error(diagnosticGapLine(record))` once inside a second try whose catch does nothing; it never throws, reads and writes no table, and itself writes no line but the gap line (E-43, D19, Sc ALL-1) - diagnosticGapCount: [extension] each error `emitDiagnostic` swallows is counted by its one gap line, in the function log's completion record of the execution that lost its record, so the number of lost records is the number of lines that contain `diagnostic gap`; a native test reads them from the `logLines` of the completion records after a mark, and a `convex-test` or pure test from the calls of a spy on `console.error`; no counter is kept in memory, because module state is not documented to survive from one function call to the next (E-43, D19, Sc ALL-1) - diagnosticNoDuration: [extension] the record carries no duration: `Date.now()` is fixed for the whole execution, as `recordedAtSource` of `spec:context.event-envelope` states, so two readings in one execution differ by nothing, and the backend records each execution's time on the execution's completion record in the function log, which also holds the execution's diagnostic line, so the design adds no second source of the same fact (E-43, D19) - diagnosticModule: [extension] `DiagnosticRecord`, `DiagnosticSink`, `consoleSink`, `limitDiagnosticBytes`, `diagnosticLine`, `diagnosticGapLine` and `emitDiagnostic` live in `src/operations/diagnostic.ts` and are exported from `src/operations/index.ts`; the audit writers stay in `src/audit/` (E-43) diff --git a/src/operations/diagnostic.ts b/src/operations/diagnostic.ts index 81b46cd..98a1426 100644 --- a/src/operations/diagnostic.ts +++ b/src/operations/diagnostic.ts @@ -78,12 +78,20 @@ function cutLine(line: string): string { export function diagnosticLine(record: DiagnosticRecord): string { const whole = recordLine(record); if (whole.bytes <= limitDiagnosticBytes) return whole.line; - // Keep the longest prefix of versions for which the line, versionsOmitted counted, fits. - for (let kept = record.versions.length - 1; kept >= 0; kept--) { + // Keep the longest prefix of versions for which the line, versionsOmitted counted, fits. A line grows + // with each version kept, so the longest fitting prefix is found by a binary search on the count. + let fitting: string | undefined; + let low = 0; + let high = record.versions.length - 1; + while (low <= high) { + const kept = Math.floor((low + high) / 2); const cut = recordLine(record, kept); - if (cut.bytes <= limitDiagnosticBytes) return cut.line; + if (cut.bytes <= limitDiagnosticBytes) { + fitting = cut.line; + low = kept + 1; + } else high = kept - 1; } - return cutLine(recordLine(record, 0).line); + return fitting ?? cutLine(recordLine(record, 0).line); } export function diagnosticGapLine(record: DiagnosticRecord): string { const fields: Record = { @@ -95,12 +103,22 @@ export function diagnosticGapLine(record: DiagnosticRecord): string { return cutLine(gapPrefix + JSON.stringify(fields)); } // Reads and writes no table. Each error the sink throws, whatever is thrown, is counted by one gap line. +// A sink is synchronous: one that returns a thenable has its record counted as lost by one gap line, +// and the thenable gets a rejection handler that does nothing, so no rejection escapes. export function emitDiagnostic( record: DiagnosticRecord, sink: DiagnosticSink = consoleSink, ): void { try { - sink(diagnosticLine(record)); + const returned: unknown = sink(diagnosticLine(record)); + if ( + (typeof returned === "object" || typeof returned === "function") && + returned !== null && + typeof (returned as { then?: unknown }).then === "function" + ) { + (returned as PromiseLike).then(undefined, () => {}); + throw new Error("A diagnostic sink is synchronous"); + } } catch { try { console.error(diagnosticGapLine(record)); diff --git a/tests/pure/diagnostic.test.ts b/tests/pure/diagnostic.test.ts index d6e0e90..9c6eb48 100644 --- a/tests/pure/diagnostic.test.ts +++ b/tests/pure/diagnostic.test.ts @@ -124,7 +124,12 @@ test("pure: versions are cut to the longest fitting prefix with versionsOmitted streamId: `subject-${i}-🦊`, version: 100 + i, })); - const input = { ...record, versions }; + // 64 more bytes of actorId make a line that leaves versionsOmitted out of the fit go over the bound. + const input = { + ...record, + actorId: record.actorId + "a".repeat(64), + versions, + }; const { versions: all, ...fields } = input; const candidate = (kept: number) => `diagnostic ${JSON.stringify({ ...fields, versionsOmitted: all.length - kept, versions: all.slice(0, kept) })}`; @@ -138,6 +143,25 @@ test("pure: versions are cut to the longest fitting prefix with versionsOmitted expect(diagnosticLine(input)).toBe(candidate(kept)); expect(bytes(candidate(kept + 1))).toBeGreaterThan(4096); }); +test("pure: at 1,000 versions the cut keeps the same prefix as a linear search down from the whole list", () => { + const versions = Array.from({ length: 1000 }, (_, i) => ({ + ...record.versions[0]!, + streamId: `subject-${i}-${"é".repeat(i % 7)}`, + version: 100 + i, + })); + const input = { ...record, versions }; + const { versions: all, ...fields } = input; + const candidate = (kept: number) => + `diagnostic ${JSON.stringify({ ...fields, versionsOmitted: all.length - kept, versions: all.slice(0, kept) })}`; + let linear = prefix(candidate(0)); + for (let kept = all.length - 1; kept >= 0; kept--) + if (bytes(candidate(kept)) <= 4096) { + linear = candidate(kept); + break; + } + expect(diagnosticLine(input)).toBe(linear); + expect(bytes(linear)).toBeLessThanOrEqual(4096); +}); test("pure: an oversized record is never dropped and the final cut preserves whole code points", () => { const input = { ...record, tenantId: "a🦊é".repeat(1600) }; const { versions, ...fields } = input; @@ -202,6 +226,32 @@ for (const thrown of [ expect(log).not.toHaveBeenCalled(); }); } +for (const [name, settle] of [ + ["rejects", () => Promise.reject(new Error("async sink broke"))], + ["resolves", () => Promise.resolve()], +] as const) + test(`pure: an async sink that ${name} counts its record as lost by one gap line, and no rejection escapes`, async () => { + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const unhandled = vi.fn(); + process.on("unhandledRejection", unhandled); + try { + // A plain function: a vi.fn spy would attach its own handler to the promise it returns. + const lines: string[] = []; + const sink = (line: string) => { + lines.push(line); + return settle(); + }; + expect(emitDiagnostic(record, sink as DiagnosticSink)).toBeUndefined(); + expect(lines).toEqual([diagnosticLine(record)]); + expect(error).toHaveBeenCalledExactlyOnceWith(diagnosticGapLine(record)); + expect(log).not.toHaveBeenCalled(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(unhandled).not.toHaveBeenCalled(); + } finally { + process.off("unhandledRejection", unhandled); + } + }); test("pure: the fixture sink faults only for the pinned tenant and otherwise uses consoleSink", () => { expect(auditFaultOrderId).toBe("order-audit-fault"); expect(sinkFaultTenant).toBe("tenant-sink-fault"); diff --git a/tests/simulator/diagnostic-support.ts b/tests/simulator/diagnostic-support.ts index 907a696..2ec87d4 100644 --- a/tests/simulator/diagnostic-support.ts +++ b/tests/simulator/diagnostic-support.ts @@ -51,7 +51,8 @@ export const publicRef = makeFunctionReference< PublicCommandArgs, CommandResponse >("diagnostic:public"); -export function entry(appended = 1, streamId = "document"): StreamDto { +// A `streams` entry an executor returns, for a stream that stood at version 100 before the command. +export function streamsEntry(appended = 1, streamId = "document"): StreamDto { const version = { tenantId: call.tenantId, contextId: "depot", @@ -102,6 +103,9 @@ export function diagnosticApp( }; const execute = vi.fn(decl.executor); decl.executor = execute; + // Two modules these tests add under their function paths, neither a file on disk: the parent's + // diagnostic module with the declaration's two entries, and the depot component's diagnosticContext + // module, which tests/simulator/diagnostic-context.ts holds. const t = convexTest(schema, { ...import.meta.glob("../../fixture/convex/**/*.ts"), "../../fixture/convex/diagnostic.ts": async () => ({ diff --git a/tests/simulator/diagnostic.test.ts b/tests/simulator/diagnostic.test.ts index a966f97..da5ed3e 100644 --- a/tests/simulator/diagnostic.test.ts +++ b/tests/simulator/diagnostic.test.ts @@ -17,7 +17,7 @@ import { issuer, internalRef, publicRef, - entry, + streamsEntry, } from "./diagnostic-support.js"; import { caught } from "./gate-support.js"; @@ -347,8 +347,8 @@ for (const failGap of [false, true]) { } for (const appended of [1, 2]) { for (const rows of [0, 1, 2]) { - test(`convex-test: ${appended} appended at version 100 and ${rows} generation rows count actual work`, async () => { - const stream = entry(appended); + test(`convex-test: a stubbed streams entry of ${appended} appended events and ${rows} generation rows written gives eventsAppended ${appended} and readModelRows ${rows}`, async () => { + const stream = streamsEntry(appended); const a = diagnosticApp({ executor: async () => ({ kind: "applied", @@ -418,8 +418,72 @@ for (const appended of [1, 2]) { }); } } +// readModelRows of spec:operations.baseline-operations counts rows inserted, replaced or deleted. +test("convex-test: a generation whose live update writes no row is left out of readModelRows", async () => { + const stream = streamsEntry(1); + const a = diagnosticApp({ + executor: async () => ({ + kind: "applied", + result: null, + versions: [stream.version], + streams: [stream], + }), + readModels: [ + { + readModel: documentSummary, + source: { contextId: "depot", streamType: "document" }, + }, + ], + }); + await a.grant(); + await a.t.run(async (ctx) => { + for (const generation of [1, 2]) + await ctx.db.insert("generations", { + readModel: "documentSummary", + generation, + projectionVersion: 1, + state: generation === 1 ? "active" : "building", + pauseRequired: false, + fence: 0, + startedAt: 0, + startedBy: "operator", + changedAt: 0, + changedBy: "operator", + }); + // The active generation has the row; the building one has none, so a live update skips it. + await ctx.db.insert("documentSummaries", { + tenantId: call.tenantId, + generation: 1, + key: "document", + projectionVersion: 1, + sourceVersions: [{ ...stream.version, version: 100 }], + documentId: "document", + status: "draft", + title: "Before", + }); + }); + const response = await a.t.mutation(internalRef, call); + expect.soft(a.records()).toEqual([ + { + ...fields, + kind: "applied", + replayed: false, + operationId: response.operationId, + eventsAppended: 1, + readModelRows: 1, + versions: [stream.version], + }, + ]); + const stored = await a.stored(); + expect(stored.rows).toHaveLength(1); + expect(stored.rows[0]).toMatchObject({ + generation: 1, + title: "Title", + sourceVersions: [stream.version], + }); +}); test("convex-test: eventsAppended sums all streams rather than counting streams or final versions", async () => { - const streams = [entry(1, "first"), entry(2, "second")]; + const streams = [streamsEntry(1, "first"), streamsEntry(2, "second")]; const a = diagnosticApp({ executor: async () => ({ kind: "businessFailure", @@ -516,7 +580,8 @@ for (const appended of [1, 2]) { }); } -// Step 10's order is observable even though either wrong order could roll back cleanly. +// At step 10 the receipt is inserted before the audit record, and the diagnostic sink is called after +// both: a proxy over db.insert and the sink write one list in the order the three happen. test("convex-test: the receipt insert finishes before audit, and both finish before the diagnostic sink", async () => { const writes: string[] = []; const a = diagnosticApp({ From ebdef9fc1f3143439b90b175d149b954ef7bad6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:41:01 +0200 Subject: [PATCH 142/155] build: close fold-in F2 F3 F10 F24 The history fold's pure test folds every interleaving in every one of the 32 contiguous splits, replaces an allocation on a second StockAllocated from the same stock item, and leaves a prior row unchanged under events it does not name. The Sc ALL-1 verifiers share BaselineOperationsWorld, and the control command runs in the after step of its own; both pass on the native backend. Co-Authored-By: Claude Opus 5.5 --- tests/native/baseline-operations-steps.ts | 32 +++++--- ...ine-operations.broken-audit-aborts.test.ts | 36 ++++---- ...rations.broken-metrics-never-abort.test.ts | 36 ++++---- tests/pure/order-allocation-history.test.ts | 82 ++++++++++++++++--- 4 files changed, 130 insertions(+), 56 deletions(-) diff --git a/tests/native/baseline-operations-steps.ts b/tests/native/baseline-operations-steps.ts index 7d5525a..1a5859c 100644 --- a/tests/native/baseline-operations-steps.ts +++ b/tests/native/baseline-operations-steps.ts @@ -15,7 +15,7 @@ import { fixtureBackend, required } from "../../harness/native.js"; import { classifyThrown } from "../../src/command/index.js"; type Answer = FunctionReturnType; -export interface AuditWorld { +export interface BaselineOperationsWorld { backend?: Backend; client?: ReturnType; tenantId?: string; @@ -33,7 +33,7 @@ async function tables(backend: Backend) { audits: await backend.admin.readTable("auditRecords"), }; } -export async function prepare(world: AuditWorld, audit: string) { +export async function prepare(world: BaselineOperationsWorld, audit: string) { expect(audit).toBe("a mandatory audit record"); const backend = await fixtureBackend(); world.backend = backend; @@ -58,7 +58,7 @@ export async function prepare(world: AuditWorld, audit: string) { }); } } -export function select(world: AuditWorld, subsystem: string) { +export function select(world: BaselineOperationsWorld, subsystem: string) { expect(["mandatory audit", "metrics and logging"]).toContain(subsystem); world.tenantId = subsystem === "mandatory audit" ? "tenant-all-1" : sinkFaultTenant; @@ -66,7 +66,7 @@ export function select(world: AuditWorld, subsystem: string) { subsystem === "mandatory audit" ? auditFaultOrderId : "order-all-1"; } async function send( - world: AuditWorld, + world: BaselineOperationsWorld, tenantId: string, orderId: string, requestKey: string, @@ -102,7 +102,7 @@ async function send( completion: required(records.find(own), "PlaceOrder completion"), }; } -export async function run(world: AuditWorld) { +export async function run(world: BaselineOperationsWorld) { world.before = await tables(required(world.backend, "backend")); Object.assign( world, @@ -114,7 +114,10 @@ export async function run(world: AuditWorld) { ), ); } -export async function assertResult(world: AuditWorld, result: string) { +export async function assertResult( + world: BaselineOperationsWorld, + result: string, +) { const after = await tables(required(world.backend, "backend")); if (result === "rolls back") { expect(world.answer).toBeUndefined(); @@ -164,7 +167,10 @@ export async function assertResult(world: AuditWorld, result: string) { }), ]); } -export async function assertFailure(world: AuditWorld, surfaced: string) { +export async function assertFailure( + world: BaselineOperationsWorld, + surfaced: string, +) { const record = required(world.completion, "completion"); const diagnostics = record.logLines.filter((line) => line.includes("diagnostic {"), @@ -195,6 +201,12 @@ export async function assertFailure(world: AuditWorld, surfaced: string) { expect(gaps[0]).toContain("applied"); expect(diagnostics).toEqual([]); } +} +// After the bound steps: on the same deployment a healthy command in tenant-all-1 commits with its +// one audit record and its one diagnostic line, so the audit writer and the sink still run. +export async function sendControl(world: BaselineOperationsWorld) { + const backend = required(world.backend, "backend"); + const before = (await tables(backend)).audits; const control = await send( world, "tenant-all-1", @@ -204,10 +216,8 @@ export async function assertFailure(world: AuditWorld, surfaced: string) { expect(control.error).toBeUndefined(); expect(control.answer).toMatchObject({ kind: "applied", replayed: false }); const operationId = (control.answer as { operationId: string }).operationId; - const audits = (await tables(required(world.backend, "backend"))).audits; - expect(audits).toHaveLength( - surfaced === "returned to the caller as a technical failure" ? 1 : 2, - ); + const audits = (await tables(backend)).audits; + expect(audits).toHaveLength(before.length + 1); expect(audits.filter((row) => row.operationId === operationId)).toEqual([ expect.objectContaining({ operationId, decision: "applied" }), ]); diff --git a/tests/native/baseline-operations.broken-audit-aborts.test.ts b/tests/native/baseline-operations.broken-audit-aborts.test.ts index e57c67d..76c7037 100644 --- a/tests/native/baseline-operations.broken-audit-aborts.test.ts +++ b/tests/native/baseline-operations.broken-audit-aborts.test.ts @@ -11,25 +11,29 @@ import { run, assertResult, assertFailure, - type AuditWorld, + sendControl, + type BaselineOperationsWorld, } from "./baseline-operations-steps.js"; const anchor = specTest({ id: testAnchorId("test:operations.baseline-operations.broken-audit-aborts"), verifies: ref("spec:operations.baseline-operations.broken-audit-aborts"), }); void anchor; -bindExample(contract, (): AuditWorld => ({}), { - "a valid command whose use case emits diagnostics and writes {audit}": async ( - world, - { audit }, - ) => prepare(world, audit), - "the {subsystem} subsystem is broken by fault injection": ( - world, - { subsystem }, - ) => select(world, subsystem), - "the command runs": async (world) => run(world), - "the command {result}": async (world, { result }) => - assertResult(world, result), - "the failure is {surfaced}": async (world, { surfaced }) => - assertFailure(world, surfaced), -}); +bindExample( + contract, + (): BaselineOperationsWorld => ({}), + { + "a valid command whose use case emits diagnostics and writes {audit}": + async (world, { audit }) => prepare(world, audit), + "the {subsystem} subsystem is broken by fault injection": ( + world, + { subsystem }, + ) => select(world, subsystem), + "the command runs": async (world) => run(world), + "the command {result}": async (world, { result }) => + assertResult(world, result), + "the failure is {surfaced}": async (world, { surfaced }) => + assertFailure(world, surfaced), + }, + sendControl, +); diff --git a/tests/native/baseline-operations.broken-metrics-never-abort.test.ts b/tests/native/baseline-operations.broken-metrics-never-abort.test.ts index 50e57c5..9fcc591 100644 --- a/tests/native/baseline-operations.broken-metrics-never-abort.test.ts +++ b/tests/native/baseline-operations.broken-metrics-never-abort.test.ts @@ -11,7 +11,8 @@ import { run, assertResult, assertFailure, - type AuditWorld, + sendControl, + type BaselineOperationsWorld, } from "./baseline-operations-steps.js"; const anchor = specTest({ id: testAnchorId( @@ -22,18 +23,21 @@ const anchor = specTest({ ), }); void anchor; -bindExample(contract, (): AuditWorld => ({}), { - "a valid command whose use case emits diagnostics and writes {audit}": async ( - world, - { audit }, - ) => prepare(world, audit), - "the {subsystem} subsystem is broken by fault injection": ( - world, - { subsystem }, - ) => select(world, subsystem), - "the command runs": async (world) => run(world), - "the command {result}": async (world, { result }) => - assertResult(world, result), - "the failure is {surfaced}": async (world, { surfaced }) => - assertFailure(world, surfaced), -}); +bindExample( + contract, + (): BaselineOperationsWorld => ({}), + { + "a valid command whose use case emits diagnostics and writes {audit}": + async (world, { audit }) => prepare(world, audit), + "the {subsystem} subsystem is broken by fault injection": ( + world, + { subsystem }, + ) => select(world, subsystem), + "the command runs": async (world) => run(world), + "the command {result}": async (world, { result }) => + assertResult(world, result), + "the failure is {surfaced}": async (world, { surfaced }) => + assertFailure(world, surfaced), + }, + sendControl, +); diff --git a/tests/pure/order-allocation-history.test.ts b/tests/pure/order-allocation-history.test.ts index 1b99f0b..269db31 100644 --- a/tests/pure/order-allocation-history.test.ts +++ b/tests/pure/order-allocation-history.test.ts @@ -152,26 +152,82 @@ test("pure: folding one order's events in every interleaving of its three stream ); }); -test("pure: folding one order's events in one call, one call per event, or two calls split anywhere gives the same row", () => { +// Every way to cut a sequence into contiguous calls: one split per subset of the gaps between events. +function splits(events: readonly T[]): T[][][] { + const gaps = events.length - 1; + const result: T[][][] = []; + for (let mask = 0; mask < 2 ** Math.max(gaps, 0); mask += 1) { + const calls: T[][] = [[]]; + events.forEach((event, at) => { + calls.at(-1)!.push(event); + if (at < gaps && (mask >> at) & 1) calls.push([]); + }); + result.push(calls); + } + return result; +} + +test("pure: folding one order's events split into contiguous calls in every way gives the same row", () => { for (const events of interleavings(streams)) { - const oneByOne = events.reduce( - (row, event) => projectOrderAllocationHistory(tenantId, row, [event]), - null, - ); - expect(oneByOne).toStrictEqual(expected); - for (let cut = 1; cut < events.length; cut += 1) { - const head = projectOrderAllocationHistory( - tenantId, - null, - events.slice(0, cut), - ); + const ways = splits(events); + // 2 ** 5 ways to cut six events, from one call to one call per event. + expect(ways).toHaveLength(32); + for (const calls of ways) { + expect(calls.flat()).toStrictEqual(events); expect( - projectOrderAllocationHistory(tenantId, head, events.slice(cut)), + calls.reduce( + (row, call) => projectOrderAllocationHistory(tenantId, row, call), + null, + ), ).toStrictEqual(expected); } } }); +test("pure: a second StockAllocated from the same stock item replaces that stock item's allocation", () => { + const reallocated = stock( + "s-1", + 4, + "StockAllocated", + { orderId: "o-1", quantity: 4 }, + 300, + ); + const replaced: OrderAllocationHistoryFields = { + ...expected, + allocations: [ + { stockItemId: "s-1", quantity: 4, allocatedAt: 300 }, + expected.allocations[1]!, + ], + }; + expect( + projectOrderAllocationHistory(tenantId, expected, [reallocated]), + ).toStrictEqual(replaced); + expect( + projectOrderAllocationHistory(tenantId, null, [ + ...orderStream, + ...ofOrder(firstStock), + reallocated, + ...ofOrder(secondStock), + ]), + ).toStrictEqual(replaced); +}); + +test("pure: an event the fold does not name, folded into a prior row, changes nothing", () => { + const unrelated = [ + firstStock[0]!, + stock("s-1", 4, "baseline", { state: {}, stateSchemaVersion: 2 }, 300), + order(3, "OrderNoted", { note: "n" }, 301), + stock("s-3", 2, "AllocationReleased", { orderId: "o-1", quantity: 1 }, 302), + ]; + for (const event of unrelated) + expect( + projectOrderAllocationHistory(tenantId, expected, [event]), + ).toStrictEqual(expected); + expect( + projectOrderAllocationHistory(tenantId, expected, unrelated), + ).toStrictEqual(expected); +}); + test("pure: a cancelled order keeps its row, and a release after the cancel folds into it", () => { const [placed, cancelled] = orderStream; const [, allocated, released] = firstStock; From d9278f8fd0d5a523b7b03f844785fae6f3465f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:51:30 +0200 Subject: [PATCH 143/155] build: close fold-in F31 F36 F37 F42 redactError redacts an error's name as its message and finds a key in a cause or in an AggregateError's errors, walked to the same bounded depth; pure tests plant a key in each. The hosted pure test asserts that the socket client the admin access builds gets the redacting logger and that a socket log line carrying the planted key comes out redacted. A pure test runs scripts/acceptance.mjs on a temporary corpus and asserts exit 1 for a record that fails a scenario and 0 for one that fails none. Co-Authored-By: Claude Opus 5.5 --- harness/child.ts | 20 +++-- tests/pure/acceptance.test.ts | 135 +++++++++++++++++++++++++++++++++- tests/pure/child.test.ts | 46 +++++++++++- tests/pure/hosted.test.ts | 75 ++++++++++++++++++- 4 files changed, 267 insertions(+), 9 deletions(-) diff --git a/harness/child.ts b/harness/child.ts index 02423d5..1da2695 100644 --- a/harness/child.ts +++ b/harness/child.ts @@ -38,7 +38,8 @@ export function redact(text: string, secrets: readonly string[]): string { return result; } // Everything of an error that a reporter or a log can print: its name, message and stack, its own -// properties and its causes. +// properties, its cause and the errors an AggregateError holds, each walked to the same depth, so +// that a cycle ends. function printable(error: unknown, depth = 0): string { const text = (value: unknown) => { try { @@ -58,13 +59,20 @@ function printable(error: unknown, depth = 0): string { const parts = [error.name, error.message, error.stack ?? ""]; for (const [key, value] of Object.entries(error)) parts.push(key, text(value), json(value)); - if (error.cause !== undefined && depth < 3) - parts.push(printable(error.cause, depth + 1)); + if (depth < 3) { + if (error.cause !== undefined) + parts.push(printable(error.cause, depth + 1)); + const nested: unknown = (error as { errors?: unknown }).errors; + if (Array.isArray(nested)) + for (const inner of nested as unknown[]) + parts.push(printable(inner, depth + 1)); + } return parts.join("\n"); } // An error that carries no secret anywhere it can be printed is returned as it is, so that a caller -// can still tell it by identity. One that carries a secret is replaced by an Error of the same name -// whose message and stack have every secret replaced, and which keeps none of its properties. +// can still tell it by identity. One that carries a secret is replaced by an Error whose name, +// message and stack have every secret replaced, and which keeps none of its properties, its cause +// and its nested errors included. export function redactError( error: unknown, secrets: readonly string[], @@ -76,7 +84,7 @@ export function redactError( const original = error instanceof Error ? error : undefined; const replaced = new Error(redact(original?.message ?? String(error), keys)); if (original !== undefined) { - replaced.name = original.name; + replaced.name = redact(original.name, keys); replaced.stack = redact( original.stack ?? `${original.name}: ${original.message}`, keys, diff --git a/tests/pure/acceptance.test.ts b/tests/pure/acceptance.test.ts index a1e3296..0bb107c 100644 --- a/tests/pure/acceptance.test.ts +++ b/tests/pure/acceptance.test.ts @@ -3,7 +3,15 @@ import { specTest, testAnchorId, } from "@libar-dev/software-delivery-protocol"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { execFile } from "node:child_process"; +import { + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { expect, test } from "vitest"; @@ -839,3 +847,128 @@ test("pure: acceptanceRows names the rows the doc's table gives Layer 0, 1 or 2, .map((scenario) => scenario.example), ).toEqual([]); }); + +// scripts/acceptance.mjs derives the graph of the repository it sits in. Run through a link with the +// link's path kept, it sits in a temporary repository of one required row and one example, so its +// build writes nothing in this tree; the corpus walk skips the links to the harness and node_modules. +test("pure: the acceptance script exits 1 on a record that fails a scenario and 0 on one that fails none", async () => { + const directory = await mkdtemp(join(tmpdir(), "libar-acceptance-script-")); + try { + for (const path of ["scripts", "design/specs", "tests"]) + await mkdir(join(directory, path), { recursive: true }); + await symlink( + join(root, "scripts/acceptance.mjs"), + join(directory, "scripts/acceptance.mjs"), + ); + for (const path of ["harness", "node_modules"]) + await symlink(join(root, path), join(directory, path)); + await writeFile( + join(directory, "design/specs/first-experiment.sdp.md"), + [ + "---", + `id: ${firstExperimentSpec}`, + "kind: workflow", + "altitude: feature", + "readiness: idea", + "relations: {}", + "---", + "# First experiment", + "", + "## Intent", + "", + "- outcome: The rows the acceptance check requires.", + "", + "## Design", + "", + "- acceptanceRows: the one required row is Sc L0-1", + "", + ].join("\n"), + ); + await writeFile( + join(directory, "design/specs/first-experiment.sample.sdp.md"), + [ + "---", + `id: ${firstExperimentSpec}.sample`, + "kind: example", + "altitude: story", + "readiness: idea", + "relations:", + ` refines: ${firstExperimentSpec}`, + ` verifies: ${firstExperimentSpec}`, + "---", + "# A sample scenario", + "", + "Sc L0-1 · pure test tier.", + "", + "## Intent", + "", + "- outcome: The sample passes.", + "", + "```gwt", + "Given a sample", + "When it runs", + "Then it passes {passed: true}", + "```", + "", + ].join("\n"), + ); + await writeFile( + join(directory, "tests/sample.test.ts"), + [ + 'import { ref, specTest, testAnchorId } from "@libar-dev/software-delivery-protocol";', + "const anchor = specTest({", + ' id: testAnchorId("test:application.first-experiment.sample"),', + ` verifies: ref("${firstExperimentSpec}.sample"),`, + "});", + "void anchor;", + "", + ].join("\n"), + ); + const run = async (result: "passed" | "failed") => { + const record = join(directory, `${result}.json`); + await writeFile( + record, + JSON.stringify({ + commit: "abc", + clean: true, + result: "passed", + startedAt: "2026-01-01T00:00:00.000Z", + tests: [ + { + project: "pure", + name: "sample", + file: "tests/sample.test.ts", + result, + backends: [], + }, + ], + }), + ); + return new Promise<{ code: number; last: string }>((resolve) => + execFile( + process.execPath, + [ + "--preserve-symlinks", + "--preserve-symlinks-main", + join(directory, "scripts/acceptance.mjs"), + record, + ], + { cwd: directory, env: { PATH: process.env.PATH ?? "" } }, + (error, stdout) => + resolve({ + code: error === null ? 0 : Number(error.code), + last: stdout.trim().split("\n").at(-1) ?? "", + }), + ), + ); + }; + const failed = await run("failed"); + expect(failed.last).toMatch(/^acceptance: not passed · 0 passed, 1 failed/); + expect(failed.code).toBe(1); + const passed = await run("passed"); + expect(passed.last).toMatch(/^acceptance: passed · 1 passed, 0 failed/); + expect(passed.code).toBe(0); + } finally { + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/tests/pure/child.test.ts b/tests/pure/child.test.ts index c812d93..240da48 100644 --- a/tests/pure/child.test.ts +++ b/tests/pure/child.test.ts @@ -6,7 +6,12 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { expect, onTestFinished, test, vi } from "vitest"; -import { redact, redactedBuffer, runChild } from "../../harness/child.js"; +import { + redact, + redactedBuffer, + redactError, + runChild, +} from "../../harness/child.js"; const secret = "0123456789abcdef-instance-secret"; test("pure: a failed child's error holds its exit code and its output, and neither its arguments nor a secret", async () => { const error: unknown = await runChild( @@ -404,3 +409,42 @@ test("pure: an abort reason that carries a secret is redacted, and one that carr expect((error as Error).stack).not.toContain(secret); } }); + +// Everything a reporter or a log prints of an error: String, its stack and its JSON. +const printedOf = (error: unknown) => + [String(error), (error as Error).stack ?? "", JSON.stringify(error)].join( + "\n", + ); +test("pure: redactError replaces a secret in the error's name as in its message", () => { + const error = new Error("plain"); + error.name = `Key${secret}Error`; + const redacted = redactError(error, [secret]); + expect(redacted).not.toBe(error); + expect((redacted as Error).name).toBe("Key[redacted]Error"); + expect((redacted as Error).message).toBe("plain"); + expect(printedOf(redacted)).not.toContain(secret); +}); +test("pure: redactError finds a secret in an AggregateError's errors and in a cause, and keeps neither", () => { + const aggregate = new AggregateError( + [new Error(`connect ${secret}`)], + "fetch failed", + ); + const caused = new Error("outer", { + cause: new Error("middle", { cause: `inner ${secret}` }), + }); + for (const error of [aggregate, caused]) { + const redacted = redactError(error, [secret]); + expect(redacted).not.toBe(error); + expect((redacted as Error).name).toBe(error.name); + expect((redacted as Error).message).toBe(error.message); + expect((redacted as Error).cause).toBeUndefined(); + expect((redacted as { errors?: unknown }).errors).toBeUndefined(); + expect(printedOf(redacted)).not.toContain(secret); + } +}); +test("pure: redactError's walk of causes and nested errors is bounded, so a cycle without a secret comes back as it is", () => { + const cycle = new AggregateError([], "cycle"); + cycle.errors.push(cycle); + cycle.cause = cycle; + expect(redactError(cycle, [secret])).toBe(cycle); +}); diff --git a/tests/pure/hosted.test.ts b/tests/pure/hosted.test.ts index 14b0038..823641f 100644 --- a/tests/pure/hosted.test.ts +++ b/tests/pure/hosted.test.ts @@ -36,6 +36,37 @@ import { hostedTarget, } from "../../harness/hosted.js"; import setupHostedRun, { crossesUtcDay } from "../../harness/hosted-run.js"; +// The socket client the admin access builds when no test replaces its factory, standing in for +// Convex's BaseConvexClient: it keeps the options it was built with and answers every subscription. +const builtSockets = vi.hoisted( + () => [] as { options: { logger?: unknown } }[], +); +vi.mock("convex/browser", async (importOriginal) => { + const actual = await importOriginal(); + class BaseConvexClient { + options: { logger?: unknown }; + onTransition: (tokens: string[]) => void; + constructor( + _url: string, + onTransition: (tokens: string[]) => void, + options: { logger?: unknown }, + ) { + this.onTransition = onTransition; + this.options = options; + builtSockets.push(this); + } + setAdminAuth() {} + subscribe() { + queueMicrotask(() => this.onTransition(["token"])); + return undefined; + } + localQueryResultByToken() { + return []; + } + async close() {} + } + return { ...actual, BaseConvexClient }; +}); // Binds this file to the harness Spec, whose rules for a native run on a hosted deployment it // checks with no network and no deploy key. const anchor = specTest({ @@ -425,13 +456,19 @@ test("pure: a planted deploy key appears in no error, output, log line or record throw new TypeError(`fetch failed: ${echo}`); return new Response(echo, { status: 500 }); }; - const socket: SocketFactory = (_url, onTransition): AdminSocketClient => ({ + // The stub socket logs through the logger it is given, as Convex's client logs a function's lines. + const socket: SocketFactory = ( + _url, + onTransition, + logger, + ): AdminSocketClient => ({ setAdminAuth() {}, subscribe() { queueMicrotask(() => onTransition(["token"])); return undefined; }, localQueryResultByToken() { + logger.log(`socket log line: ${echo}`); throw new Error(`the socket answered: ${echo}`); }, async close() {}, @@ -544,6 +581,13 @@ test("pure: a planted deploy key appears in no error, output, log line or record expect(String((failure as Error).message)).toContain("[redacted]"); expect(JSON.stringify(outputs[1])).toContain("[redacted]"); expect(printed.join("\n")).toContain("[redacted]"); + expect( + printed.filter((line) => line.startsWith("socket log line: ")), + ).not.toEqual([]); + for (const line of printed.filter((one) => + one.startsWith("socket log line: "), + )) + expect(line).toContain("[redacted]"); expect(recordText).toContain("[redacted]"); const json = (value: unknown) => { try { @@ -569,6 +613,35 @@ test("pure: a planted deploy key appears in no error, output, log line or record ).toEqual([]); }); +test("pure: the hosted kind's socket client gets the redacting logger, so a socket log line carrying a planted key comes out redacted", async () => { + const key = plantedKey(); + const forms = deployKeyForms(key); + const { access } = createHostedAccess({ + variables: variables(key), + composition: await temporaryComposition(), + home: await temporary("libar-hosted-home-"), + state: { deployed: undefined, environment: new Set() }, + }); + builtSockets.length = 0; + expect(await access.environment()).toEqual({}); + expect(builtSockets).toHaveLength(1); + const logger = builtSockets[0]!.options.logger as child.ClientLogger; + const printed: string[] = []; + const spy = vi + .spyOn(console, "log") + .mockImplementation((...args: unknown[]) => { + printed.push(format(...args)); + }); + try { + logger.log(`socket log line: ${forms.join(" ")}`); + } finally { + spy.mockRestore(); + } + expect(printed).toHaveLength(1); + expect(printed[0]).toMatch(/^socket log line: .*\[redacted\]/); + for (const form of forms) expect(printed[0]).not.toContain(form); +}); + test("pure: the run setup reads the version and the usage before the run, and after it amends the record in one step", async () => { const key = plantedKey(); const echo = deployKeyForms(key).join(" "); From 804aa51f34dc559bacb5f74bdc23a1b47c03b552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:52:14 +0200 Subject: [PATCH 144/155] build: close fold-in F26 F27 F28 F29 F30 F32 F33 F35 F38 F39 F40 F41 The hosted in-place example says the five states are planted before the export and read after the import from an archive with no scheduler table, and that the restored pending reference is cancelled; its test compares the documents and the scheduler rows of every scope the copy mounts, a fixed list. Probe 3 asserts the documents each call read, the usage settles against a 300 s deadline, and the retention example checks each age against the reading's time. The five hosted examples join the foundation pack, the README states 237 Specs, and the harness Spec names the deploy key on a hosted deployment, allows a same-run before and after comparison, and says where the hosted examples run. Co-Authored-By: Claude Opus 5.5 --- design/README.md | 4 +- ...ions.probe-7-hosted-in-place-import.sdp.md | 4 +- design/specs/foundation.pack.sdp.md | 5 ++ design/specs/platform/native-harness.sdp.md | 8 +-- tests/hosted/examples.ts | 54 +++++++++++++------ tests/hosted/hosted-examples.test.ts | 23 ++++---- 6 files changed, 63 insertions(+), 35 deletions(-) diff --git a/design/README.md b/design/README.md index b2a8701..52db332 100644 --- a/design/README.md +++ b/design/README.md @@ -2,7 +2,7 @@ ## What this is -The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 232 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. +The decisions document at [`../docs/convex-transactional-domain-platform-decisions.md`](../docs/convex-transactional-domain-platform-decisions.md) is the source of truth for what the platform is: nineteen decisions, twelve laws, a ledger of the Convex facts they rest on, seven probes and forty-three acceptance scenarios. This directory is the design written down as a Libar Software Delivery Protocol corpus: 237 Specs in five packs, each Spec a Markdown carrier under `specs/`, each claim cited back to the doc or marked as an extension the doc does not make. The Design Review that `sdp view` writes to `generated/design-review/` is the rendering of the corpus for a human reader. The plan the authoring agents followed is [`PLAN.md`](PLAN.md); it fixes IDs, layout and conventions and stays binding for anyone who edits a Spec. Work on the corpus continues in sessions. [`SESSIONS.md`](SESSIONS.md) is the protocol, [`STATE.md`](STATE.md) says where the work stands, and [`ROADMAP.md`](ROADMAP.md) orders the next units. The thesis, in the doc's words. Ownership follows bounded contexts. Atomicity follows the business operation. Asynchrony follows a concrete need to defer work. The default operation is one Convex mutation that authorizes a command, makes the domain decision, saves current state with its events, updates essential read models and records the outcome. A second transaction exists only to wait, to spread load, or to reach an external system. The design succeeds when adding domain sophistication does not add infrastructure. @@ -45,7 +45,7 @@ python3 design/tools/check.py Expected result on the integrated corpus: ``` -237 specs · 5 packs · 142 anchors → 384 nodes · 1246 edges (0 errors, 0 warnings) +237 specs · 5 packs · 142 anchors → 384 nodes · 1251 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings (conformance + honesty over the one graph) ``` diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.sdp.md index 325a7d9..9034af0 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import.sdp.md @@ -17,7 +17,7 @@ Probe 7 · native backend tier · temporary copy of the fixture composition · h ```gwt Given the hosted deployment running the hosted driver's temporary copy of the fixture composition, which adds the scheduler references and reactions of the local in-place example -When it exports business documents and five scheduler states with the deploy key, adds documents and schedules, then imports the backup archive in place before pending reactions are due +When it plants business documents and five scheduler states, exports with the deploy key a backup archive that holds no scheduler table, adds documents and schedules, imports the backup archive in place before pending reactions are due, and reads the five states after the import Then documents including ids and creation times in {scopes: "parent and every mounted component"} equal the exported documents {equal: true} And every scheduler row in each scope is unchanged {unchanged: true} And later business changes are gone {removed: true} and the destination environment is unchanged {environmentUnchanged: true} @@ -30,5 +30,5 @@ And each restored scheduler reference resolves to the kept row {found: true} and - Runs second in the hosted driver, after F16's retention example; it starts no local backend. - The copy is the one the retention example ran on, so the table `plantedSchedules` stays in the export and the import restores it as it was at the export. - The bound values are what the local backend showed on F12; a hosted answer that differs is a finding on F12. -- The test changes business data before the import and fails if it does not reach that boundary; every kept pending schedule is let run before the test ends, so that no later example finds it pending. +- The test changes business data before the import and fails if it does not reach that boundary; the restored pending reference is cancelled, and every other kept schedule is let run before the test ends, so that no later example finds it pending. - Hosted dashboard restore is not run: the deploy key reaches the deployment through the CLI's export and import only. The fresh-destination case needs a second deployment and is not run. diff --git a/design/specs/foundation.pack.sdp.md b/design/specs/foundation.pack.sdp.md index 8368cc6..5ffda67 100644 --- a/design/specs/foundation.pack.sdp.md +++ b/design/specs/foundation.pack.sdp.md @@ -11,6 +11,8 @@ specs: - spec:platform.native-harness.ordinary-client-refused-internal-function - spec:platform.native-harness.ordinary-client-refused-component-function - spec:platform.native-harness.admin-key-acting-as-identity-reaches-internal-function + - spec:platform.native-harness.undeclared-table-survives-another-composition-deploy + - spec:platform.native-harness.hosted-deployment-accepts-production-composition - spec:platform.existing-systems - spec:laws.law01-sanctioned-writes-only - spec:laws.law02-state-and-events-commit-together @@ -51,6 +53,7 @@ specs: - spec:facts.f04-nested-calls-cost-more-than-helpers - spec:facts.f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-mutation - spec:facts.f04-nested-calls-cost-more-than-helpers.probe-3-component-call-from-query + - spec:facts.f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls - spec:facts.f05-react-client-retries-until-confirmed - spec:facts.f05-react-client-retries-until-confirmed.probe-1-client-closed-with-pending-mutation - spec:facts.f05-react-client-retries-until-confirmed.probe-1-backend-restart-with-pending-mutation @@ -64,6 +67,7 @@ specs: - spec:facts.f12-backups-exclude-pending-scheduled-functions - spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-fresh-import - spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import + - spec:facts.f12-backups-exclude-pending-scheduled-functions.probe-7-hosted-in-place-import - spec:facts.f13-transactions-have-limits - spec:facts.f13-transactions-have-limits.probe-4-nested-call-shares-limits - spec:facts.f13-transactions-have-limits.probe-4-component-call-shares-limits @@ -84,6 +88,7 @@ specs: - spec:facts.f16-scheduled-functions-table-shows-failed-runs - spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-scan - spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-states + - spec:facts.f16-scheduled-functions-table-shows-failed-runs.probe-7-hosted-retention - spec:facts.f17-migrations-fits-generation-backfill - spec:facts.f17-migrations-fits-generation-backfill.probe-6-batch-races-command - spec:facts.f17-migrations-fits-generation-backfill.probe-6-interruption-and-resume diff --git a/design/specs/platform/native-harness.sdp.md b/design/specs/platform/native-harness.sdp.md index bb2dfd9..666dc86 100644 --- a/design/specs/platform/native-harness.sdp.md +++ b/design/specs/platform/native-harness.sdp.md @@ -33,7 +33,7 @@ The doc says that native tests prove component and nested-mutation behavior, sch - [extension] A composition that runs natively reads its issuer, its application ID and its key set from the deployment's environment variables, and the harness sets the key set to a data URI that holds the fixture issuer's public key (E-13, S13) - [extension] No caller under test carries the admin key, alone or acting as an identity, because a call made with it passes Convex's visibility check (E-13) - [extension] The fixture issuer is the identity source of a native run and its one named difference from production; it is recorded as the identity source and is not an adjusted configuration (E-13, Sc L2-9) -- [extension] The harness uses the admin key to deploy, to generate code, to set and read environment variables, to read stored data and the function log, to run fixture functions, and to call a composition's internal functions that admin access reaches, its grant mutation and its operator entries (E-15, E-48) +- [extension] The harness uses the admin key on the local backend, and the deploy key on a hosted deployment, to deploy, to generate code, to set and read environment variables, to read stored data and the function log, to run fixture functions, and to call a composition's internal functions that admin access reaches, its grant mutation and its operator entries (E-15, E-48) - [extension] On the local backend, every Convex CLI call the harness makes names the backend it started on the command line and gets an environment of its own, so that no variable of the caller's shell and no dotenv file selects another deployment (E-15) - [extension] No generated secret, the instance secret or the admin key, reaches an error message, test output or a record (E-15) - [extension] A native run on a hosted deployment uses the same harness, run record and compositions as one on the local backend and names the hosted deployment as the backend it ran on; it claims no parity with a release under Sc L2-9, passes no scenario row, and shows only what the local backend cannot: the deploy key's authority, the deployment's acceptance of a composition, its reachability, its report of its own usage and what it keeps between runs (E-59, Sc L2-9) @@ -42,7 +42,7 @@ The doc says that native tests prove component and nested-mutation behavior, sch - [extension] A native run on a hosted deployment starts no local backend: `fixtureBackend` and `productionBackend` refuse inside it (E-59) - [extension] The deploy key enters the harness as one process variable and reaches a Convex CLI child only as `CONVEX_DEPLOY_KEY` in the environment the harness gives that child, never as an argument; the harness's HTTP and socket clients carry it in a header or an authentication message, never in a URL (E-59, E-15) - [extension] No form of the deploy key, the whole key, its part after the `|` or either of them URL-encoded, reaches an error message, test output, a record or an artifact of the continuous integration; every member of the admin access to a hosted deployment and every CLI child replaces those forms in what it throws before it throws, where the response is first seen, and the record is redacted again as a whole before it is written (E-59, E-15) -- [extension] The hosted deployment keeps its data between runs, so the record of a native run on it states a kept dataset, `kept from earlier runs`, and an example on it asserts only on what its own run wrote or on what a table of the run's temporary copy says an earlier run planted (E-59) +- [extension] The hosted deployment keeps its data between runs, so the record of a native run on it states a kept dataset, `kept from earlier runs`, and an example on it asserts only on what its own run wrote or on what a table of the run's temporary copy says an earlier run planted, and a comparison of the same documents read before and after an operation of its own run, which assumes nothing about their content, is allowed (E-59) - [extension] Native runs on the hosted deployment run one at a time, which the harness cannot enforce across machines, so the record names whether the continuous integration or a developer ran it; a failed native run on the hosted deployment fails no check of a commit's continuous integration (E-59) - [extension] The record of a native run on a hosted deployment holds the deployment's usage readings before and after the run, with the time of each, and no example asserts a cost from them; a difference of two readings is the run's own only when no other run on the deployment overlapped it, and is unestablished when that cannot be shown; the plan it names is the stated plan, and the backend version is what the deployment reports or `null`, never mapped to a release (E-59) - [extension] An assertion about authority, identity or visibility is made through an ordinary client (E-15, E-13) @@ -76,7 +76,7 @@ The harness is plain test support with no Convex function in it. The entries who - backend: [extension] the disposable backend is the `convex-local-backend` binary of one pinned release, checked against a recorded hash; the harness starts it on local ports with fresh storage, a generated instance secret and the admin key the binary derives from it, sets the environment variables, runs the Convex CLI in the composition's project directory to deploy it, and at the end of the test stops the process and removes the storage (E-15, S13) - identity: [extension] a caller is a `ConvexHttpClient` or a `ConvexClient` whose `setAuth` carries a token the harness signs; `auth.config.ts` declares one `customJwt` provider whose `issuer`, `applicationID` and `jwks` are environment variables; the token carries `iss`, `aud`, `sub`, `iat` and `exp` and no other claim, so `ctx.auth.getUserIdentity()` in a public entry returns the identity those claims yield, and `establishActor`, the grants read and the namespace rules run unchanged as long as they read nothing a provider adds beyond them (E-13, S13, D11) -- adminReads: [extension] the harness reads a table of the app or of a component, and the function log's completion records, with the admin key; a table read returns every document with every Convex value intact or fails, never a sample, and is several queries, not one snapshot; a log read returns every completion record after a point the test marks or fails, and a mark taken before a restart cannot be read after it; a completion record carries at least `identifier`, `componentPath`, `requestId`, `executionTime` in seconds, `cachedResult`, `error` and `logLines` as an array of strings (E-15) +- adminReads: [extension] the harness reads a table of the app or of a component, and the function log's completion records, with the admin key on the local backend and the deploy key on a hosted deployment; a table read returns every document with every Convex value intact or fails, never a sample, and is several queries, not one snapshot; a log read returns every completion record after a point the test marks or fails, and a mark taken before a restart cannot be read after it; a completion record carries at least `identifier`, `componentPath`, `requestId`, `executionTime` in seconds, `cachedResult`, `error` and `logLines` as an array of strings (E-15) - typeFixtureIssuer: [extension] `interface FixtureIssuer { issuer: string; applicationID: string; jwks: string; token(subject: string): Promise }` (E-13) - typeBackend: [extension] `interface Backend { url: string; adminKey: string; issuer: FixtureIssuer; admin: AdminAccess; stop(): Promise; kill(): Promise; restart(): Promise; dispose(): Promise; facts(): BackendFacts }`; `stop` and `kill` keep the storage; `dispose` alone removes it and is final, so a disposed backend cannot be started again; `adminKey` is there for the one example that shows why no caller carries it (E-15) - typeAdminAccess: [extension] `interface AdminAccess { deploy(): Promise; deployTemporary(directory: string, signal?: AbortSignal): Promise<{ command: string[]; stdout: string; stderr: string }>; codegen(): Promise; setEnvironment(variables: Record): Promise; environment(): Promise>; run(name: string, args?: Record, options?: { component?: string }): Promise; readTable(table: string, options?: { component?: string; pageSize?: number }): Promise[]>; writeTable(table: string, write: { insert: Record } | { patch: string; fields: Record } | { delete: string }, options?: { component?: string }): Promise; logMark(): Promise; completionsSince(mark: LogMark, until: (records: readonly CompletionRecord[]) => boolean, timeoutMs?: number): Promise; exportBackupArchive(path: string): Promise; importBackupArchive(path: string): Promise }` holds every use of the admin key and takes no identity; the two backup archive members run the Convex CLI's export of a backup archive and its import with replacement against this backend, and a test uses them and never the admin key itself; `deployTemporary` deploys a temporary copy, stops its deploy child when the caller's signal or the backend's own aborts, returns its redacted command and clears the plain composition claim (E-15) @@ -101,7 +101,7 @@ The harness is plain test support with no Convex function in it. The entries who - deployKeyActions: [extension] the deploy key allows exactly `deployment:deploy`, `deployment:env:view`, `deployment:env:write`, `deployment:data:view`, `deployment:data:write`, `deployment:functions:runInternalQueries`, `deployment:functions:runInternalMutations`, `deployment:functions:runInternalActions`, `deployment:logs:view`, `deployment:usage:view`, `deployment:backups:view`, `deployment:backups:create`, `deployment:backups:download` and `deployment:backups:import`, the actions the routes of `AdminAccess` and the CLI's export and import check in the upstream backend at `5c7cb5b`, and not `deployment:functions:actAsUser`, pause, usage limits, integrations or the audit log (E-59) - hostedRedaction: [extension] the secrets of a native run on a hosted deployment are the four forms of the deploy key, given to every child, every buffer and the reporter; the admin access built over a hosted deployment catches every rejection of its members, the HTTP client's raw response text and its `Invalid response` text among them, a fetch that rejects and a call that aborts, and rethrows it with those forms replaced; its HTTP and socket clients take a logger that replaces them; a CLI child's output is redacted on success as on failure and on an abort; the reporter redacts the serialized record before it writes it; the import upload token, the export download token, application data and the fixture issuer's private key are not among these secrets and are named as not covered, not scrubbed as if they were (E-59, E-15) - hostedConfiguration: [extension] `npm run test:hosted` runs `vitest run --config vitest.hosted.config.ts`, whose one project, `hosted`, runs one driver file under `tests/hosted/` with its own global setup and the evidence reporter; the driver runs four examples in order: Probe 7's retention example before any deploy or import, Probe 7's replacement in place, the production composition's acceptance, and Probe 3's quota half; the first, second and fourth run on the one temporary copy of the fixture composition that `hostedProbeCopy` describes, and the fourth leaves that copy deployed for the next run; the retention example deploys the copy after its read and, when the deployment held no planting record, records that it read nothing; `npm run test:all` never includes it (E-59) -- hostedSchedule: [extension] the workflow `.github/workflows/hosted.yml` runs on `workflow_dispatch` and on a schedule at 03:00 UTC on Monday and Thursday; GitHub runs a schedule from the default branch, and the job runs in a GitHub environment named `hosted` that holds the key and admits the default branch only, so a dispatch from another branch gets no key; in one concurrency group that cancels no running job; the run's output goes to a file, and the job prints that captured log and uploads the record as the artifact `hosted-evidence` only after `scripts/hosted-record-check.mjs`, reading the key from its own environment, finds no form of it in the record or the captured log, and no other workflow waits on it; until the workflow is on the default branch, the owner's machine runs `npm run test:hosted` with the variables in its shell, and its record names a developer as the one who ran it (E-59) +- hostedSchedule: [extension] the workflow `.github/workflows/hosted.yml` runs on `workflow_dispatch` and on a schedule at 03:00 UTC on Monday and Thursday; GitHub runs a schedule from the default branch, and the job runs in a GitHub environment named `hosted` that holds the key and admits the default branch only, so a dispatch from another branch gets no key; in one concurrency group that cancels no running job; the run's output goes to a file, and the job prints that captured log and uploads the record as the artifact `hosted-evidence` only after `scripts/hosted-record-check.mjs`, reading the key from its own environment, finds no form of it in the record or the captured log, and no other workflow waits on it; the hosted examples run through this workflow, whose record names continuous integration as the one who ran them, or through `npm run test:hosted` in a shell that holds the variables, whose record names a developer (E-59) - hostedRedactionTest: [extension] one pure test with no network plants a key in the form of a development deploy key, gives the hosted admin access an injected fetch and a stub socket and runs a stub CLI script, each of which echoes the key in all four forms on every path, calls every member so that each fails, captures `process.stdout` and `process.stderr` and the reporter's output, writes a record into a temporary directory, and fails when any form appears in any of them (E-59) - hostedProbeCopy: [extension] the temporary copy of the fixture composition that the hosted driver deploys adds, from `tests/hosted/planting.ts` and the local in-place example's own scheduler module, the table `plantedSchedules` of planting records, one for each schedule a run planted in the parent, with its scheduler ID, the state and `completedTime` the run saw, the `startedAt` of the run that planted it and the count of backup archive imports since, the module `planting` that plants one succeeded, one failed and one canceled schedule and writes their planting records, and the scheduler references and reactions of the local in-place example in the parent and in the fixture contexts `depot` and `yard`; Probe 3's parent mutation is the fixture's own `readCost:viaMutation`, whose helper and component paths read the same document; the fixture composition itself gains nothing, as `probe7Narrowed` of `spec:facts.probe-plan` requires, and a later run knows a schedule's age after its scheduler row is gone (E-59) - compositionLayout: [extension] a composition is a name, a project directory and a functions directory; the Convex CLI reads `convex.json` and `package.json` from its working directory and takes no functions directory any other way, so the harness runs every CLI call in the composition's project directory; the fixture composition's project directory is the repository root, whose `convex.json` names `fixture/convex/`, and the production composition's is `example/`, whose `convex.json` names `convex/` and whose `package.json` is a symbolic link to the root's, so the repository has one `package.json` (E-15) diff --git a/tests/hosted/examples.ts b/tests/hosted/examples.ts index 4727185..e336d85 100644 --- a/tests/hosted/examples.ts +++ b/tests/hosted/examples.ts @@ -49,6 +49,7 @@ async function deployCopy(backend: HostedBackend) { export interface RetentionWorld { backend?: HostedBackend; readNothing?: boolean; + readAt?: number; earlier?: Row[]; observations?: { scheduledId: string; @@ -81,6 +82,7 @@ export async function readThenPlant(world: RetentionWorld) { const scheduled = await backend.admin.readTable("_scheduled_functions"); const readAt = Date.now(); const present = new Set(scheduled.map((row) => String(row._id))); + world.readAt = readAt; world.earlier = earlier; world.observations = earlier.map((record) => ({ scheduledId: String(record.scheduledId), @@ -112,16 +114,21 @@ export async function readThenPlant(world: RetentionWorld) { ); measureValue("own planted schedules", world.own); } +// Each age is the reading's time minus the time its planting record holds, and null when the record +// holds none. export function earlierRecorded(world: RetentionWorld, recorded: boolean) { const earlier = required(world.earlier, "the earlier planting records"); const observations = required(world.observations, "the observations"); + const readAt = required(world.readAt, "the time of the reading"); expect( observations.length === earlier.length && observations.every( (observation) => typeof observation.read === "boolean" && - (typeof observation.ageMs === "number" || - observation.completedTime === null), + observation.ageMs === + (typeof observation.completedTime === "number" + ? readAt - observation.completedTime + : null), ), ).toBe(recorded); } @@ -167,13 +174,14 @@ async function storedData(backend: Backend, entries: string[]) { } return data; } +// The scheduler rows of the parent and of every mounted component, keyed by scope. async function schedulerRows(backend: Backend) { const rows: Record = {}; - for (const scope of copyScopes) - rows[scopeName(scope)] = await backend.admin.readTable( - "_scheduled_functions", - scopeOptions(scope), - ); + rows.parent = await backend.admin.readTable("_scheduled_functions"); + for (const component of mountedComponents) + rows[component] = await backend.admin.readTable("_scheduled_functions", { + component, + }); return rows; } // The state kinds of the scheduler rows this run's labels name, in one scope. @@ -386,8 +394,8 @@ export async function hostedInPlace( }); } measureValue("kept reactions", { due, reactions }); - // Every kept schedule has run or is released before the example ends, so that no later example - // finds one pending or in progress. + // The restored pending reference was cancelled above; every other kept schedule has run or is + // released before the example ends, so that no later example finds one pending or in progress. for (const scope of copyScopes) for (const row of await backend.admin.readTable( "schedulerData", @@ -519,21 +527,26 @@ const functionCalls = (usage: HostedUsage): number => { throw new Error("The usage reading holds no function calls for the day"); return value; }; -// Reads the usage until two readings in a row agree on the function calls, and keeps every reading. +// Reads the usage every 5 s until two readings in a row agree on the function calls, and keeps every +// reading; the deadline is 300 s from the first reading's start, and no reading starts after it. +const limitSettleMs = 300000; async function settledReading( backend: HostedBackend, readings: HostedUsage[], ): Promise { + const deadline = Date.now() + limitSettleMs; let last = await backend.admin.usage(); readings.push(last); - for (let attempt = 0; attempt < 60; attempt++) { + while (Date.now() + 5000 <= deadline) { await new Promise((resolve) => setTimeout(resolve, 5000)); const next = await backend.admin.usage(); readings.push(next); if (functionCalls(next) === functionCalls(last)) return next; last = next; } - throw new Error("Two usage readings in a row never agreed in 300 s"); + throw new Error( + `Two usage readings in a row never agreed in ${limitSettleMs / 1000} s`, + ); } export async function quotaDeployment(world: QuotaWorld, reads: number) { const backend = await hostedBackend({ deploy: false }); @@ -558,12 +571,19 @@ export async function quotaSets(world: QuotaWorld, calls: number) { for (const [index, path] of ( ["component", "helper", "component"] as const ).entries()) { + // Each call counts only when the backend's own count of the documents it read is the bound number. for (let call = 0; call < calls; call++) - await client.mutation(fixtureApi.readCost.viaMutation, { - ...seeds, - path, - reads, - cacheBuster: Date.now() * 100 + index * calls + call, + expect( + await client.mutation(fixtureApi.readCost.viaMutation, { + ...seeds, + path, + reads, + cacheBuster: Date.now() * 100 + index * calls + call, + }), + "the documents one call read", + ).toEqual({ + documentsRead: reads, + id: path === "component" ? seeds.annex : seeds.own, }); const readings: HostedUsage[] = []; const after = await settledReading(backend, readings); diff --git a/tests/hosted/hosted-examples.test.ts b/tests/hosted/hosted-examples.test.ts index 6fbaa90..cda3077 100644 --- a/tests/hosted/hosted-examples.test.ts +++ b/tests/hosted/hosted-examples.test.ts @@ -21,7 +21,6 @@ import { earlierRecorded, hostedInPlace, inPlaceDeployment, - mountedComponents, ownStates, placeOrderRecords, quotaDeployment, @@ -84,6 +83,15 @@ bindExample(probe7HostedRetentionContract, (): RetentionWorld => ({}), { ownStates(world, states, completedTime), }); +// The scopes the copy mounts: the parent and every component of the fixture composition. +const copyMounts = [ + "parent", + "annex", + "annexClock", + "migrations", + "depot", + "yard", +]; type InPlace = InPlaceWorld & { observation?: Awaited>; }; @@ -92,7 +100,7 @@ const observed = (world: InPlace) => bindExample(probe7HostedInPlaceImportContract, (): InPlace => ({}), { "the hosted deployment running the hosted driver's temporary copy of the fixture composition, which adds the scheduler references and reactions of the local in-place example": (world) => inPlaceDeployment(world), - "it exports business documents and five scheduler states with the deploy key, adds documents and schedules, then imports the backup archive in place before pending reactions are due": + "it plants business documents and five scheduler states, exports with the deploy key a backup archive that holds no scheduler table, adds documents and schedules, imports the backup archive in place before pending reactions are due, and reads the five states after the import": async (world) => { world.observation = await hostedInPlace( required(world.backend, "the hosted backend"), @@ -107,13 +115,7 @@ bindExample(probe7HostedInPlaceImportContract, (): InPlace => ({}), { const read = new Set( Object.keys(o.exported).map((key) => key.split("/")[0]), ); - expect(read).toEqual( - new Set( - ["parent", ...mountedComponents].filter((scope) => read.has(scope)), - ), - ); - for (const scope of ["parent", "depot", "yard"]) - expect(read).toContain(scope); + expect(read).toEqual(new Set(copyMounts)); expect(isDeepStrictEqual(o.restored, o.exported)).toBe(equal); }, "every scheduler row in each scope is unchanged {unchanged}": ( @@ -121,7 +123,8 @@ bindExample(probe7HostedInPlaceImportContract, (): InPlace => ({}), { { unchanged }, ) => { const o = observed(world); - expect(Object.keys(o.after).sort()).toEqual(["depot", "parent", "yard"]); + expect(Object.keys(o.before).sort()).toEqual([...copyMounts].sort()); + expect(Object.keys(o.after).sort()).toEqual([...copyMounts].sort()); for (const scope of Object.keys(o.after)) expect(isDeepStrictEqual(o.after[scope], o.before[scope])).toBe( unchanged, From 73ffe6dc25a504f22cdfacb5041247c1bf8f9e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:54:08 +0200 Subject: [PATCH 145/155] design: close fold-in, the local in-place example says what its archive holds Co-Authored-By: Claude Opus 5.5 --- ...e-pending-scheduled-functions.probe-7-in-place-import.sdp.md | 2 +- ...-pending-scheduled-functions.probe-7-in-place-import.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md index 84f6259..33de611 100644 --- a/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md +++ b/design/specs/facts/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.sdp.md @@ -17,7 +17,7 @@ Probe 7 · native backend tier · temporary copy of the production composition. ```gwt Given a production composition has temporary functions for scheduler references and reactions -When it exports business documents and five scheduler states, adds orders and schedules, then imports the backup archive in place before pending reactions are due +When it plants business documents and five scheduler states, exports a backup archive that holds no scheduler table, adds orders and schedules, then imports the backup archive in place before pending reactions are due Then documents including ids and creation times in {scopes: "parent,orders,inventory"} equal the exported documents {equal: true} And every scheduler row in each scope is unchanged {unchanged: true} And later business changes are gone {removed: true} and the destination environment is unchanged {environmentUnchanged: true} diff --git a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts index ae95328..f54e438 100644 --- a/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts +++ b/tests/native/f12-backups-exclude-pending-scheduled-functions.probe-7-in-place-import.test.ts @@ -27,7 +27,7 @@ bindExample(contract, (): World => ({}), { async (world) => { Object.assign(world, await schedulingBackend()); }, - "it exports business documents and five scheduler states, adds orders and schedules, then imports the backup archive in place before pending reactions are due": + "it plants business documents and five scheduler states, exports a backup archive that holds no scheduler table, adds orders and schedules, then imports the backup archive in place before pending reactions are due": async (world) => { world.observation = await inPlaceImport(world.backend!, world.directory!); }, From c1bdf8078ed2b767958fac3b63d9b8675acfdb4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 10:54:30 +0200 Subject: [PATCH 146/155] design: the close of round 3, OD-155 decided by the owner, OD-036 waiting for the first hosted run, the restore finding's note Co-Authored-By: Claude Opus 5.5 --- design/decisions/register.json | 49 +++++++++++++++++++++++++--- design/reviews/consensus-ledger.json | 2 +- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/design/decisions/register.json b/design/decisions/register.json index e7c3f11..7c77a94 100644 --- a/design/decisions/register.json +++ b/design/decisions/register.json @@ -1451,7 +1451,7 @@ "kind": "specQuestion", "ref": "spec:facts.probe-plan#2", "file": "design/specs/facts/probe-plan.sdp.md", - "line": 39 + "line": 44 } ], "families": [ @@ -1469,12 +1469,13 @@ "provisionalReading": null, "blocks": null, "advisor": "convex", - "status": "sorted", + "status": "waiting", "foldedInto": [], "class": "owner", - "reason": "Rule 1, money: a hosted deployment is the owner's account, and the quota half cannot run on a local backend [design/specs/facts/probe-plan.sdp.md:39]. Doubt: it is a need more than a decision; the decision it serves is OD-063.", + "reason": "Rule 1, money: a hosted deployment is the owner's account, and the quota half cannot run on a local backend [design/specs/facts/probe-plan.sdp.md:44]. Doubt: it is a need more than a decision; the decision it serves is OD-063. The account it needs is OD-155's.", "lean": { - "do": "Run Probe 3's quota half once on a hosted deployment after S3's local numbers, so that OQ1 (OD-063) has both halves." + "do": "Run Probe 3's quota half as the hosted example spec:facts.f04-nested-calls-cost-more-than-helpers.probe-3-hosted-function-calls on the one hosted deployment of E-59, once the owner provides what OD-155 names, so that OQ1 (OD-063) has both halves.", + "settles": "A run on the hosted deployment in which the helper set of calls adds exactly one function call per parent call between two component sets that add the same number, every seedStatus is complete and no UTC day boundary falls between the first and last reading; otherwise its record says the count is not settled and this row stays open." }, "checkedBy": null, "fork": null, @@ -6322,6 +6323,46 @@ "decidedBy": null, "decidedOn": null, "ruling": null + }, + { + "id": "OD-155", + "title": "The hosted arrangement and account of E-59: one shared development deployment, each developer's own, a preview per run, or a fresh development deployment per run", + "sources": [ + { + "kind": "specQuestion", + "ref": "spec:platform.native-harness#4", + "file": "design/specs/platform/native-harness.sdp.md", + "line": 27 + } + ], + "families": [ + "platform", + "facts" + ], + "cites": [ + "E-59", + "Sc L2-9", + "F4", + "F12", + "F16", + "Probe 3", + "Probe 7" + ], + "provisionalReading": "the option taken here: the same harness, run record and compositions run natively on one hosted development deployment in a team and project of their own, selected only by `npm run test:hosted`, one run at a time, from a workflow whose failure fails no other check, with one development deploy key limited to the actions `deployKeyActions` lists", + "blocks": null, + "advisor": "convex", + "status": "decided", + "foldedInto": [], + "class": "owner", + "reason": "Rule 1, money: the team, the plan, the project and the deploy key are the owner's account; raised by the hosted lane of the tooling unit and made a row by its patch.", + "lean": { + "do": "One hosted development deployment in a project of its own, in the owner's otherwise unused team on the Professional plan, with one development deploy key held as one secret by a GitHub environment that admits the default branch only, run twice a week and on dispatch." + }, + "checkedBy": null, + "fork": null, + "decidedBy": "owner", + "decidedOn": "2026-10-03", + "ruling": "The owner's words in the tooling unit, on the advisor's eight hosted recommendations: \"I agree with all recommendations. We have unused Convex Professional account. Let me know how to login.\" Taken as the lean: one hosted development deployment, dev/hosted in the project libar-platform-hosted of the owner's otherwise unused team on the Professional plan, with one development deploy key, hosted-ci-scoped, limited to the fourteen actions, held as the one secret HOSTED_DEPLOY_KEY by the GitHub environment hosted, which admits the default branch only; run twice a week and on dispatch. The owner made the key, the environment and the secret by hand on 2026-10-03." } ], "possiblySame": [ diff --git a/design/reviews/consensus-ledger.json b/design/reviews/consensus-ledger.json index 460ff60..fd3419d 100644 --- a/design/reviews/consensus-ledger.json +++ b/design/reviews/consensus-ledger.json @@ -2021,7 +2021,7 @@ "issue": "Two restore checks fail for a deployment that has ever run a meaning migration, and one of them cannot be passed at all. First, queries says a stream still awaiting its baseline is refused by `rebuild` and 'the caller counts either refusal as a journal finding', and `typeRebuildResult` names `awaitingBaseline` among 'what the restore's journal check records as findings'. Restore says 'nobody reopens a deployment that failed a check', and also that a baseline chain in flight at the backup 'is restarted by the operator after the accepted branch'. A backup taken inside a migration window therefore holds cold streams that are findings, the run can never be accepted, and nothing can clear them: commands are refused by the restore door and the baseline driver reads `gateAllows`, which answers not allowed while `MAINTENANCE_MODE` is `restore`. restore's `fnStreamCheckBatch` itself names only `tailTooLong` and `equal` false, so the two Specs also disagree on what the batch records. Second, the read-model check asserts 'a row whose source versions equal the current versions'. The journal's own risk bullet says that after a sweep 'every read-model row whose source the sweep touched has `sourceVersions` one behind its stream until the next command or verify pass rewrites it', and no verify pass runs outside a rebuild. So after any sweep every cold stream's row is a read-model finding and the restore stays closed.", "fix": "Make `awaitingBaseline` pending work, not a finding: `streamCheckBatch` counts such streams per stream type on the run record as a `pendingWork` item of a new kind `baselineMigration`, checks their event contiguity without a fold, and the accepted branch tells the operator to restart the sweep, as restore's rule already says. Correct queries' Contract bullet and `typeRebuildResult` to match. For the read-model check, compare fields with the projection of the current DTO and require the row's versions to be at most the current ones; a row that is behind only by baseline events and equal in fields is not a finding. Alternatively have the baseline driver re-project the streams it migrated, which needs `writeBaseline` to return their DTOs, and say which is taken.", "status": "open", - "note": "the platform review of 2026-10-02, APPL-09: At HEAD, context/queries.sdp.md:79 calls awaitingBaseline a journal finding, restore.sdp.md:69 restarts the sweep only \"after the accepted branch\", and journal.sdp.md:109 defers under the gate, so a mid-migration backup cannot reach acceptance. The issue holds; the fix must prove a controlled repair and acceptance sequence and must not silently relax source-version equality for a genuinely stale row. the platform review of 2026-10-02, CORE-06: At HEAD, journal.sdp.md:40 leaves sourceVersions \"one behind\" after a baseline and :109 calls maintenance directly, so a cold read model can retain old values as well as a lagging version. The issue remains; a version-only acceptance exception cannot establish equal projected meaning. The build must show: convex-test snapshots mid-migration, after migration with cold read-model rows and with genuine corruption must let the first two recover and keep the corrupt case closed. A native backend restore drill including component data must prove the controlled repair and acceptance order without silently accepting stale projected meaning.", + "note": "the platform review of 2026-10-02, APPL-09: At HEAD, context/queries.sdp.md:79 calls awaitingBaseline a journal finding, restore.sdp.md:69 restarts the sweep only \"after the accepted branch\", and journal.sdp.md:109 defers under the gate, so a mid-migration backup cannot reach acceptance. The issue holds; the fix must prove a controlled repair and acceptance sequence and must not silently relax source-version equality for a genuinely stale row. the platform review of 2026-10-02, CORE-06: At HEAD, journal.sdp.md:40 leaves sourceVersions \"one behind\" after a baseline and :109 calls maintenance directly, so a cold read model can retain old values as well as a lagging version. The issue remains; a version-only acceptance exception cannot establish equal projected meaning. The build must show: convex-test snapshots mid-migration, after migration with cold read-model rows and with genuine corruption must let the first two recover and keep the corrupt case closed. A native backend restore drill including component data must prove the controlled repair and acceptance order without silently accepting stale projected meaning. Close of round 3, from sol's read of sitting two's OD-041 ruling: list refuses a stream behind a history-reading step with writePaused, so the restore's journal check cannot enumerate a stream awaiting a baseline through list; restore.sdp.md records the question, and pendingWorkItemValidator gains the kind awaitingBaseline; the read that does not refuse is designed with the baseline sweep's entry (5a).", "slice": "S5", "detector": "convex-test" }, From 651fcc34ddccf26cfaecc5dcbb37a87bee60f9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 11:04:27 +0200 Subject: [PATCH 147/155] build: close fold-in, redaction fails closed redactError replaces an error with a redacted stub when its walk reaches the depth bound with something left to read, and walks the values of a plain object as it walks a cause and nested errors; a pure test covers each path. The sweep's byte-bound test states 8 MiB as a literal, and the acceptance script test takes 60 s, which two graph builds need under load. Co-Authored-By: Claude Opus 5.5 --- harness/child.ts | 71 +++++++++++++++++--------- tests/pure/acceptance.test.ts | 3 +- tests/pure/child.test.ts | 33 +++++++++++- tests/simulator/receipts-sweep.test.ts | 3 +- 4 files changed, 81 insertions(+), 29 deletions(-) diff --git a/harness/child.ts b/harness/child.ts index 1da2695..07eb409 100644 --- a/harness/child.ts +++ b/harness/child.ts @@ -37,42 +37,64 @@ export function redact(text: string, secrets: readonly string[]): string { } return result; } +// How deep the walk of causes, nested errors and object values goes. +const limitRedactionDepth = 3; // Everything of an error that a reporter or a log can print: its name, message and stack, its own -// properties, its cause and the errors an AggregateError holds, each walked to the same depth, so -// that a cycle ends. -function printable(error: unknown, depth = 0): string { - const text = (value: unknown) => { +// properties, and, walked to a bounded depth, its cause, the errors an AggregateError holds, the +// values of its own properties and the values of a plain object among them. cut is true when the walk +// reached its bound with something left to read, which a cycle always does. +function printable(value: unknown, depth = 0): { text: string; cut: boolean } { + const text = (one: unknown) => { try { - return typeof value === "string" ? value : String(value); + return typeof one === "string" ? one : String(one); } catch { return ""; } }; - const json = (value: unknown) => { + const json = (one: unknown) => { try { - return JSON.stringify(value) ?? ""; + return JSON.stringify(one) ?? ""; } catch { - return text(value); + return text(one); } }; - if (!(error instanceof Error)) return `${text(error)}\n${json(error)}`; - const parts = [error.name, error.message, error.stack ?? ""]; - for (const [key, value] of Object.entries(error)) - parts.push(key, text(value), json(value)); - if (depth < 3) { - if (error.cause !== undefined) - parts.push(printable(error.cause, depth + 1)); - const nested: unknown = (error as { errors?: unknown }).errors; - if (Array.isArray(nested)) - for (const inner of nested as unknown[]) - parts.push(printable(inner, depth + 1)); + const parts: string[] = []; + const inner: unknown[] = []; + if (value instanceof Error) { + parts.push(value.name, value.message, value.stack ?? ""); + for (const [key, one] of Object.entries(value)) { + parts.push(key, text(one), json(one)); + inner.push(one); + } + inner.push(value.cause); + const errors: unknown = (value as { errors?: unknown }).errors; + if (Array.isArray(errors)) inner.push(...(errors as unknown[])); + } else { + parts.push(text(value), json(value)); + if (typeof value === "object" && value !== null) + inner.push(...Object.values(value)); + } + // A value that is not an object is printed as it is; an object is walked. + const objects: object[] = []; + for (const one of inner) + if (typeof one === "object" && one !== null) objects.push(one); + else if (one !== undefined) parts.push(text(one)); + if (objects.length === 0) return { text: parts.join("\n"), cut: false }; + if (depth >= limitRedactionDepth) + return { text: parts.join("\n"), cut: true }; + let cut = false; + for (const one of objects) { + const walked = printable(one, depth + 1); + parts.push(walked.text); + cut ||= walked.cut; } - return parts.join("\n"); + return { text: parts.join("\n"), cut }; } // An error that carries no secret anywhere it can be printed is returned as it is, so that a caller -// can still tell it by identity. One that carries a secret is replaced by an Error whose name, -// message and stack have every secret replaced, and which keeps none of its properties, its cause -// and its nested errors included. +// can still tell it by identity. One that carries a secret, or whose walk reached its bound with +// something left to read, fails closed: it is replaced by an Error whose name, message and stack +// have every secret replaced, and which keeps none of its properties, its cause and its nested +// errors included. export function redactError( error: unknown, secrets: readonly string[], @@ -80,7 +102,8 @@ export function redactError( const keys = secrets.filter((secret) => secret !== ""); if (keys.length === 0) return error; const printed = printable(error); - if (!keys.some((key) => printed.includes(key))) return error; + if (!printed.cut && !keys.some((key) => printed.text.includes(key))) + return error; const original = error instanceof Error ? error : undefined; const replaced = new Error(redact(original?.message ?? String(error), keys)); if (original !== undefined) { diff --git a/tests/pure/acceptance.test.ts b/tests/pure/acceptance.test.ts index 0bb107c..50bf8e2 100644 --- a/tests/pure/acceptance.test.ts +++ b/tests/pure/acceptance.test.ts @@ -971,4 +971,5 @@ test("pure: the acceptance script exits 1 on a record that fails a scenario and } finally { await rm(directory, { recursive: true, force: true }); } -}); + // Two children that each derive a graph: more than the default 5 s on a loaded machine. +}, 60000); diff --git a/tests/pure/child.test.ts b/tests/pure/child.test.ts index 240da48..9fb3fd0 100644 --- a/tests/pure/child.test.ts +++ b/tests/pure/child.test.ts @@ -442,9 +442,38 @@ test("pure: redactError finds a secret in an AggregateError's errors and in a ca expect(printedOf(redacted)).not.toContain(secret); } }); -test("pure: redactError's walk of causes and nested errors is bounded, so a cycle without a secret comes back as it is", () => { +test("pure: redactError fails closed past its depth bound: a key four cause links down, a deep chain without one and a cycle each come back as a redacted stub", () => { + const chain = (last: string) => + new Error("one", { + cause: new Error("two", { + cause: new Error("three", { + cause: new Error("four", { cause: new Error(last) }), + }), + }), + }); const cycle = new AggregateError([], "cycle"); cycle.errors.push(cycle); cycle.cause = cycle; - expect(redactError(cycle, [secret])).toBe(cycle); + for (const error of [chain(`deep ${secret}`), chain("deep"), cycle]) { + const redacted = redactError(error, [secret]); + expect(redacted).not.toBe(error); + expect((redacted as Error).message).toBe(error.message); + expect((redacted as Error).cause).toBeUndefined(); + expect((redacted as { errors?: unknown }).errors).toBeUndefined(); + expect(printedOf(redacted)).not.toContain(secret); + } + const shallow = new Error("one", { cause: new Error("two") }); + expect(redactError(shallow, [secret])).toBe(shallow); +}); +test("pure: redactError walks the values of a plain object in a cause, so a key in an error held there is found and dropped", () => { + const error = new Error("outer", { + cause: { error: new Error(`held ${secret}`) }, + }); + const redacted = redactError(error, [secret]); + expect(redacted).not.toBe(error); + expect((redacted as Error).message).toBe("outer"); + expect((redacted as Error).cause).toBeUndefined(); + expect(printedOf(redacted)).not.toContain(secret); + const clean = new Error("outer", { cause: { error: new Error("held") } }); + expect(redactError(clean, [secret])).toBe(clean); }); diff --git a/tests/simulator/receipts-sweep.test.ts b/tests/simulator/receipts-sweep.test.ts index 8c5fd77..ac291ed 100644 --- a/tests/simulator/receipts-sweep.test.ts +++ b/tests/simulator/receipts-sweep.test.ts @@ -1,7 +1,6 @@ import { convexTest, type TestConvex } from "convex-test"; import type { SchemaDefinition } from "convex/server"; import { getDocumentSize } from "convex/values"; -import { limitSweepBytes } from "../../src/command/index.js"; import { commandTables } from "../../src/command/tables.js"; import { expect, test } from "vitest"; import schema from "../../fixture/convex/schema.js"; @@ -172,7 +171,7 @@ for (const composition of compositions) { for (const row of rows) { bytes += getDocumentSize(row); read++; - if (bytes >= limitSweepBytes) break; + if (bytes >= 8 * 1024 * 1024) break; } expect(read).toBeLessThan(rows.length); expect( From 1b3ff5841af46e6c78e8a6e6fd91e6d14e9c5cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 11:09:26 +0200 Subject: [PATCH 148/155] design: the close of round 3 and of the feature branch STATE.md describes the branch at its close: round 3's mini roadmap done but for the hosted tier's first runs, which are the owner's after the merge; the one pull request into main; what each lane, the integration, the review, the fold-in and its read showed; the code as it is; decision 42 amended and 52 added; the owner queue and the leads brought to the present, with CI's failure at d2a8d1b recorded. ROADMAP.md marks 4a, 4c, 4d and 4f done and the hosted tier built. Co-Authored-By: Claude Opus 5.5 --- design/ROADMAP.md | 36 ++++----- design/STATE.md | 197 ++++++++++++++++++++++------------------------ 2 files changed, 113 insertions(+), 120 deletions(-) diff --git a/design/ROADMAP.md b/design/ROADMAP.md index 5952789..3bf2057 100644 --- a/design/ROADMAP.md +++ b/design/ROADMAP.md @@ -1,6 +1,6 @@ # Roadmap: the next units -Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of round 3's sitting two on 2026-10-03. +Written on 2026-10-02 by the main thread, from four advisor memos written the same day under `design/advisors/task-roadmap.md`. It is a work document: it orders units and names what each needs and shows, and it says nothing a Spec should say. The owner rules on it. The memos, the plants and the check are in the roadmap unit's folder, which the project notes name. It is kept current at each close, last at the close of round 3 on 2026-10-03. ## How to read it @@ -25,14 +25,14 @@ The slices of `STATE.md` keep their names. S3 is round 1's attribution and round | 2 | g | Also done in round 2: the review's fold-in, the caller's signal in the harness, three `CancelOrder` tests, `fillTenants` and the resumed batch size on paper | mixed | operator, domain | under an evening | the review | | 3 | a | The gate, the restore door, and a gate change as audit that fails closed. Done in round 2: 106 proof tests written apart from the fix, all passing at the merge, the fix wrong nowhere | mixed | operator | an evening | 2b | | 3 | b | The online rebuild of the order summary, Sc L2-5 at the native tier: thirteen functions per composition, laid out in `layout-3b.md` of the tooling unit as three lanes and 33 proof obligations | build | domain | a night | 2b, 2d, 3a, the layout's eight questions ruled, the five register rows recorded | -| 3 | e | The hosted tier, commissioned from `design-hosted-tier.md` of the tooling unit: four hosted examples on one shared development deployment, the local backend primary | build | convex, operator | unknown until the first run | 3b, the deploy key and its secret (done), sitting two's words (done), `patch-hosted-specs.diff` applied with its two word changes | +| 3 | e | The hosted tier, commissioned from `design-hosted-tier.md` of the tooling unit: four hosted examples on one shared development deployment, the local backend primary. Built in the close of round 3; its first runs are the owner's after the merge | build | convex, operator | unknown until the first run | 3b, the deploy key and its secret (done), sitting two's words (done), `patch-hosted-specs.diff` applied with its two word changes | | 3 | c | The two harness reads and the fact rows no Spec carries. Done in round 2: F18 to F20 probed, the "no job" assertions read `_scheduled_functions`; the deployed-function read stays deferred | mixed | convex | a session plus an evening | nothing | | 3 | d | Sitting two: the language, the pass sentence, and the S5 rows. Done: sixteen rows ruled from one recommendations page after one Opus and one astra read | decide | product, domain, operator | an evening | nothing | -| 4 | a | A command's audit fails closed and its diagnostic never aborts, Sc ALL-1 | mixed | operator | an evening | 3a | -| 4 | c | The history view's fold, its memory and its rollback, on paper | design | domain | an evening | 2c | -| 4 | d | The receipts sweep, one tenant per run | build | operator | under an evening | the tenant list from 2b | +| 4 | a | A command's audit fails closed and its diagnostic never aborts, Sc ALL-1. Done in the close of round 3: both Sc ALL-1 examples pass natively, the proof written apart from the fix | mixed | operator | an evening | 3a | +| 4 | c | The history view's fold, its memory and its rollback, on paper. Done in the close of round 3: ten choices ruled, ten ledger findings on paper, one pure fold with its test | design | domain | an evening | 2c | +| 4 | d | The receipts sweep, one tenant per run. Done in the close of round 3: `receipts.sweep` and `sweepNext` in both compositions, at `convex-test` | build | operator | under an evening | the tenant list from 2b | | 4 | e | `ready` on the built Specs, by family | decide | product | a session, one owner unit | 1c, sitting two | -| 4 | f | The language in the code: the refusal field `entry`, the tier names in the examples' opening lines and the harness parser, the harness's backup archive names | build | product | under an evening | sitting two | +| 4 | f | The language in the code: the refusal field `entry`, the tier names in the examples' opening lines and the harness parser, the harness's backup archive names. Done in the close of round 3 | build | product | under an evening | sitting two | | 5 | a | Baselines, migration and the fold bound, Sc L2-7 at the native tier; the history-reading sweep with its transient refusal and the restore check's pending-work inventory, at `convex-test` for the interleavings and natively for the sweep | mixed | domain | a night or more | OD-041 (ruled), 3b | | 5 | b | The restore procedure and the drill, Sc L2-8 | mixed | operator | unknown until 4b | 4b, 5a | | 5 | c | The first adopter-facing cut: README, the example walkthrough, the package shape | mixed | product | a night | OD-047, OD-048, Sc L2-5 to L2-8 bound | @@ -68,14 +68,14 @@ Done on 2026-10-02 and 2026-10-03, as nine lanes merged into `feature/round-2`, ## Round 3, as it now stands -The owner discusses the next session's work before it is prepared. In this order: +Every item is done but the hosted tier's first runs, which are the owner's after the merge of `feature/round-2`. `STATE.md`, "Where the work stands", has the numbers of each. The owner discusses the next session's work before it is prepared. -1. **3b, the online rebuild of the order summary**, is done, on 2026-10-03: thirteen functions per composition, every read model installed through its first rebuild, the parent list paged across a switch, and Sc L2-5 passing at the native tier on the production composition. `STATE.md`, "Where the work stands", has its numbers: 325 of 325 proof tests at the merge, the L2-5 run's times, the review and three reads of the fold-in. -2. **The hosted tier**, a build unit commissioned from `design-hosted-tier.md` of the tooling unit, is next. Its prerequisites are met but one: the environment `hosted` with its three variables and the deploy key `hosted-ci-scoped`, scoped to the fourteen actions and checked, exist since 2026-10-03; sitting two ruled the words; `patch-hosted-specs.diff` is applied first in the unit, with its "marker" read as planting record and its test "pass" as test result. The one thing still the owner's is the dashboard check: `clean-capybara-16` is a production deployment the design does not want. It shows the four hosted examples run by the workflow on `main`: F16's retention, F12's in-place import, the production composition's acceptance and Probe 3's function calls. The local backend stays primary. -3. **3d, sitting two**, is done, on 2026-10-03: sixteen rows ruled (the five language rows with OD-017 and OD-110, OD-006 with OD-008, OD-138, OD-041, OD-042, OD-114, OD-044, OD-063 and OD-048), each applied at the Spec that owns it, `CONTEXT.md` carrying the words. The method: one recommendations page by the main thread, one Opus and one astra read of it, then the ruling. Two doc hunks wait for the owner's hand, and the language in the code is lane 4f. -4. **4a, command audit and the diagnostic.** 3a has built `writeAudit` and step 10, so what is left is the Spec text on which outcomes emit a record and where a duration comes from, the sink, and Sc ALL-1 bound. -5. **4c, the history view on paper.** 2c is done, so `OrderCancelled` and `AllocationReleased` exist for its fold. -6. **4d, the receipts sweep.** The tenant list and `nextTenant` are built. +1. **3b, the online rebuild of the order summary**, is done, on 2026-10-03: thirteen functions per composition, every read model installed through its first rebuild, the parent list paged across a switch, and Sc L2-5 passing at the native tier on the production composition; `STATE.md` has its numbers. +2. **The hosted tier** is built, in the close of round 3: steps 1 to 9 of section 7 of `design-hosted-tier.md`, redaction and the record's target at the pure tier, the undeclared table's example on the local backend, the four hosted examples written and compiled. Its first runs, from the owner's machine and then by the workflow on `main`, and the dashboard check of `clean-capybara-16` are the owner's after the merge; `STATE.md` has the rest. +3. **3d, sitting two**, is done, on 2026-10-03: sixteen rows ruled, each applied at the Spec that owns it, `CONTEXT.md` carrying the words; two doc hunks wait for the owner's hand, and the language in the code, lane 4f, is done in the close of round 3; `STATE.md` has its numbers. +4. **4a, command audit and the diagnostic**, is done, in the close of round 3: the Spec text ruled, the code and its proof written apart from it, both Sc ALL-1 examples passing at the native tier; `STATE.md` has its numbers. +5. **4c, the history view on paper**, is done, in the close of round 3: ten choices ruled and one pure fold with its test, the build left to 5d; `STATE.md` has its numbers. +6. **4d, the receipts sweep**, is done, in the close of round 3: `receipts.sweep` and `sweepNext` in both compositions, bounded by 1,000 rows and 8 MiB, at `convex-test`; `STATE.md` has its numbers. ## Round 3 @@ -83,7 +83,7 @@ The owner discusses the next session's work before it is prepared. In this order **3b. The online rebuild.** The rebuild's thirteen functions per composition (eight operator entries, four batches and `getGenerations`, as `layout-3b.md` counts them) and the progress row, with the version-selected projection of 2b. A version 2 of the order summary built as generation 2 while `PlaceOrder` runs, a fence bump as the interrupt, verify, switch, intervening writes, rollback, verify again, switch back. Sc L2-5 on the native backend with a record. Built on 2026-10-03 as round 3's unit 3b, with generation 2 at version 1 and the two versions on the fixture's `documentTitle` at `convex-test`, as Q1 of its rulings says. -**The hosted tier.** Built from `design-hosted-tier.md` section 1, decision 7, in its order: redaction first, then the record shape and the acceptance filter, `hostedTarget`, the selection kinds, the run setup, the local example of the undeclared table, the copy's modules, the four examples and the workflow. It is built when one record carries `target: "hosted deployment"` with no form of the key, the acceptance example passed, and a CI run on `main` says so. Previews, a second deployment, deletion and cost in money are out. The project and the deployment `dev/hosted` exist since 2026-10-03. +**The hosted tier.** Built from `design-hosted-tier.md` section 1, decision 7, in its order: redaction first, then the record shape and the acceptance filter, `hostedTarget`, the selection kinds, the run setup, the local example of the undeclared table, the copy's modules, the four examples and the workflow. It is built when one record carries `target: "hosted deployment"` with no form of the key, the acceptance example passed, and a CI run on `main` says so. Previews, a second deployment, deletion and cost in money are out. The project and the deployment `dev/hosted` exist since 2026-10-03. Steps 1 to 9 were built in the close of round 3, and the design's eight "built when" items wait for the owner's first run after the merge. **3c. The harness reads and the fact rows.** A read of `_scheduled_functions` through admin access, so that the "no job" assertions of Sc L2-2 and L2-3 close on the table and not on a window; three fact Specs written from the pinned package source and a probe on the pinned release, and not from the leads that point at them: the commit timestamp as OD-044's lean says, the time budget nested calls draw on, and the late byte check of `convex-helpers`; the scheduled-argument size discrepancy recorded on F13; one probe counting component calls per mutation on the pinned release. The native read of the deployed function list is deferred: OD-021's lean takes the committed generated api as the pinned list, and the main thread judges that compiled check sufficient until a scenario needs more. @@ -91,11 +91,11 @@ The owner discusses the next session's work before it is prepared. In this order ## Round 4 -**4a. Command audit and the diagnostic.** `audit` on the declaration, `writeAudit` at step 10, `emitDiagnostic` at the end of the mutation through a sink that swallows every error and counts it, the two injection points in the fixture composition only, and both examples of Sc ALL-1 bound. The Spec text first says which outcomes emit a record and where a duration comes from. +**4a. Command audit and the diagnostic.** `audit` on the declaration, `writeAudit` at step 10, `emitDiagnostic` at the end of the mutation through a sink that swallows every error and counts it, the two injection points in the fixture composition only, and both examples of Sc ALL-1 bound. The Spec text first says which outcomes emit a record and where a duration comes from. Done in the close of round 3, the proof written apart from the fix. -**4c. The history view on paper.** One row whatever the order across streams, fold memory beyond a deleted row, rollback of a paused view as a refusal or a re-closing of the sources, the cross-context row narrowed to D8's parent query, the aggregate write split on a key change. Four ledger findings fixed on paper and one pure determinism test. The build of the view waits for a view that needs it. +**4c. The history view on paper.** One row whatever the order across streams, fold memory beyond a deleted row, rollback of a paused view as a refusal or a re-closing of the sources, the cross-context row narrowed to D8's parent query, the aggregate write split on a key change. Four ledger findings fixed on paper and one pure determinism test. The build of the view waits for a view that needs it. Done in the close of round 3, with ten findings fixed or partially fixed on paper. -**4d. The receipts sweep.** One internal mutation per tenant over the expiry index, at most the bound, returning what it deleted and whether more remains, with the loop over tenants from 2b's ruling. +**4d. The receipts sweep.** One internal mutation per tenant over the expiry index, at most the bound, returning what it deleted and whether more remains, with the loop over tenants from 2b's ruling. Done in the close of round 3, as `receipts.sweep` and `sweepNext`. **4e. `ready` by family.** The fork for OD-024 lists the built Specs with their floors and their open questions and asks for `ready` by family: kernel and context first, then command, then application without the generation registry. A Spec with an unruled `owner` row stays `defined`. @@ -110,7 +110,7 @@ The owner discusses the next session's work before it is prepared. In this order ## Deferred, with the trigger - Layers 3 to 6: at their trigger, after Probe 7, as the doc and tactical decision 3 say. OD-050 to OD-054 and OD-056 stay waiting until an activation record names a consumer. -- Probe 3's quota half, OD-036: no longer deferred. It is the fourth example of the hosted tier, the unit after 3b. +- Probe 3's quota half, OD-036: no longer deferred. It is the fourth example of the hosted tier, built in the close of round 3, and OD-036 waits for its first run. - The aggregate form: at a view that needs it, OD-075. The history view is no longer deferred: ruling F10 keeps it in the first experiment, 4c writes its rules and 5d builds it. - Receipt tombstones, the derived mapping and `streamParts`, the import command: when a command needs them, OD-100, OD-113, OD-072. - OD-092's `transactionLimits` probe: before any budget or `maxStreams` is raised; nothing in rounds 1 to 5 raises one. diff --git a/design/STATE.md b/design/STATE.md index fa674a2..69309f7 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -1,40 +1,53 @@ # State of the design corpus -Written on 2026-10-03 at the close of sitting two, unit 3d of round 3 of the roadmap, which followed unit 3b, the tooling unit, round 2, round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. +Written on 2026-10-03 at the close of round 3 and of `feature/round-2`, which followed sitting two (round 3's unit 3d), unit 3b, the tooling unit, round 2, round 1, the roadmap unit, the decision register unit, the platform review unit and slice S2. `SESSIONS.md` says how to use this file. Every close rewrites it. `ROADMAP.md` orders the next units and is kept current at each close. ## Measured at close ``` -232 specs · 5 packs · 131 anchors → 368 nodes · 1225 edges (0 errors, 0 warnings) +237 specs · 5 packs · 142 anchors → 384 nodes · 1251 edges (0 errors, 0 warnings) validate: 0 errors · 0 warnings; readiness divergence: [] -open questions: 69 Specs, 147 questions, 9 blocking; extensions registered: 53 -stated readiness: {'defined': 222, 'idea': 1, 'scoped': 9} -ledger: 278 findings, {'fixed': 152, 'open': 94, 'owner': 1, 'partially-fixed': 31} -open findings by slice: {'L3': 30, 'P': 29, 'S3': 1, 'S4': 15, 'S5': 19} -corpus digest: 94bb9c63bfd80580 +open questions: 69 Specs, 153 questions, 9 blocking; extensions registered: 54 +stated readiness: {'defined': 227, 'idea': 1, 'scoped': 9} +ledger: 278 findings, {'fixed': 159, 'open': 80, 'owner': 1, 'partially-fixed': 38} +open findings by slice: {'L3': 30, 'P': 29, 'S3': 1, 'S4': 4, 'S5': 16} +corpus digest: 2851beee8cf5338b protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` -`python3 design/tools/decisions.py`, after sitting two's patch of sixteen rows: +`python3 design/tools/decisions.py`, after this unit's two register commits, `19738b4` and `c1bdf80`: ``` -total: 154 -status: unsorted=0, sorted=82, waiting=20, decided=37, folded=15 -class: tactical=3, delegated=86, owner=50, null=15 -advisor: convex=36, domain=45, operator=34, product=39 +total: 155 +status: unsorted=0, sorted=80, waiting=21, decided=39, folded=15 +class: tactical=3, delegated=86, owner=51, null=15 +advisor: convex=37, domain=45, operator=34, product=39 ``` -**Round 2, the tooling unit and round 3's unit 3b are on `feature/round-2`, not on `main`.** `main` is at `592f1dc`, the close of round 1. The lanes of round 2 and of 3b branched from `main` or from `feature/round-2` and were merged into `feature/round-2`, which is pushed. CI passed on its pushes at `1b92bd8` and `a5d339d` in round 2, and at `314c265` (7 m 5 s), `2283083` (7 m 22 s) and `de285bb` (7 m 17 s) in 3b. CI runs again on the push of this close. The clean whole-tier run of 3b is CI's: the fold-in's two local whole-tier runs ran while another project's vitest job held the load average at 20 to 24, the first failed two timing-sensitive tests and the second four, and each failed file passed alone. No pull request to `main` is opened until the owner has reviewed the branch. The owner's words: "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue" and "We can discuss details before preparation of work for the next session." Tactical decision 42 holds them. The next lanes branch from `feature/round-2` and merge back into it. +**Round 2, the tooling unit, round 3's unit 3b, sitting two and the close of round 3 are on `feature/round-2`, not on `main`.** `main` is still at `592f1dc`, the close of round 1. The close of round 3 did the rest of the mini roadmap (4f, the hosted tier, 4c, 4a and 4d) and ends the branch: `feature/round-2` goes to `main` as one pull request, which the main thread opens at this close for the owner's review and merge. The owner's words at the opening of the close: "forks have been resolved and mini roadmap unlocked. please complete the work to wrap up the feature branch and PR. use Opus, GPT 6 Astra and 6.1 Sol and orchestrate from the main thread". The main thread was Opus 5.5; no Fable agent and no advisor ran. Its head is this close's commit, on top of the fold-in's follow-up `651fcc3`. Tactical decision 42 holds the branch and its one pull request. -The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Each unit works on its own branch, named for the unit, so that `main` stays green. The owner gave the session a mandate on 2026-10-01 to merge at this early stage, and said the same day that a pull request per unit was not a good idea while the design is iterated quickly. So from slice S1 on there is no pull request: CI runs on every push, and the session fast-forwards `main` once CI is green on the last commit. Slice S0 was the one unit merged through a pull request. That held until the merge of 2026-10-02. +CI passed on the branch's pushes in round 2 (`1b92bd8`, `a5d339d`) and in 3b (`314c265`, `2283083`, `de285bb`, and its close `840b32c` in 7 m 12 s). It failed on the push of sitting two, `d2a8d1b` (run 37105650743): two native files of 166, Probe 4's nested-call example and Sc L2-5's `rebuild.online-rebuild-interrupt-resume`, whose `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` (`OptimisticConcurrencyControlFailure`), so acceptance read 31 passed, 1 failed and 6 missing (Leads 43). Nothing was pushed after it until this close. The local whole-tier runs of this unit ran under load and are no proof of the whole tier: CI's run on the push of this close, and on the pull request, is. -**Slice S2 and the platform review unit are on `main`.** The owner said on 2026-10-02, after the close of the platform review unit: "Please merge to main and push. This is the last change we will do without formal PRs since things are starting to shape up." The session fast-forwarded `main` to `fix/platform-review`, which holds `slice/s2`, and pushed it. The owner's earlier bar for S2 was "high-confidence changes only", with a revert to `9221da1` kept cheap until the owner had seen the result. - -From this merge on, a unit reached `main` through a pull request, tactical decision 32, as round 1's four lanes did. Round 2 reaches it as one branch the owner reviews first, decision 42. The review as it was received is in commit `70c15bc`, with paths of the owner's machine in its links. The files were deleted from the tree at the close, and the owner said the same day that the deleted review may stay in the history. +The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Slice S0 reached it through a pull request; from S1 on the session fast-forwarded it once CI was green, on the owner's mandate of 2026-10-01, until the merge of slice S2 and the platform review unit on 2026-10-02, when the owner said: "Please merge to main and push. This is the last change we will do without formal PRs since things are starting to shape up." Round 1's four lanes then reached it through pull requests #6 to #9 (decision 32). The platform review as it was received is in commit `70c15bc`, with paths of the owner's machine in its links; its files left the tree at that unit's close, and the owner said the deleted review may stay in the history. ## Where the work stands -Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, round 1 of that roadmap is on `main`, round 2 is on `feature/round-2`, a design unit for the tooling followed on the same branch, and so did round 3's first unit, 3b, the online rebuild of the order summary, and then sitting two, the owner's decisions of round 3. +Slices S0, S1 and S2 are built, one unit after them took in an independent review of the whole, one built the decision register and the advisor panel, one asked the panel for a roadmap, and round 1 of that roadmap is on `main`. On `feature/round-2` follow round 2, a design unit for the tooling, round 3's unit 3b (the online rebuild of the order summary), sitting two (the owner's decisions of round 3) and the close of round 3, which did the rest of round 3's mini roadmap. + +The close of round 3, `application-platform-round-3-close`, on 2026-10-03 from about 09:15 to 11:15, on the ThinkPad, on `feature/round-2` from `d2a8d1b`. The cut was the list "Round 3, as it now stands" of `ROADMAP.md`, whose items 1 and 3 (3b, sitting two) were done: the hosted tier, 4a, 4c, 4d and lane 4f, which sitting two created, then an integration, a review, one fold-in, one read of it and this close. Five lanes ran at once, each in a worktree from `d2a8d1b`: language (4f, an Opus agent), hosted (an Opus lead), history (4c on paper, an Opus agent), audit (4a, an Opus designer, then an Opus builder with astra writing the proof apart) and sweep (4d, sol). `rulings-1-close.md` and `foldin-list.md` in the unit folder hold every ruling. What each showed, at the tier named: + +- The leans. `gpt-6.1-sol` at medium (`job-mus2kbol-a1d74933`, 2.4 min) checked three delegated leans on a copy with one planted citation and caught the plant. OD-149 held, with corrections that touch only its reasons and narrow its downside: `decided` by its operator advisor's checked lean. OD-095 held 6 of 6, but its lean is a `do` alone and the tool applies only a lean in full, so it stays `sorted` with the check recorded, and the sweep was built on the Spec text. OD-135 held 9 of 19 (line drift, a `details` field that does not exist, "nothing is built" stale): `sorted` with the check recorded, the audit lane's Spec text the provisional reading. Committed as `19738b4`. +- The sweep (4d). Sol at medium (`job-mus2kbog-3e1ce6e6`, 2.6 and 0.6 min over two turns) built `receipts.sweep` as `fnSweep` pins it, with `sweepNext` and a cursor `after` the caller holds, registered in both compositions and on no command path; nothing schedules it. Ruled: compaction waits for tombstones (OD-100), so `compacted` stays 0; an expired tombstone row is deleted, not refused (sent back once); a bad `limit` or `now` is a plain error. `convex-test`. +- The language (4f). An Opus agent, 14.6 min over two turns (`f179f6b`, 151 files): `entry` on the refusal in 24 library lines, one fixture line and about 110 test lines; the four tier names in 103 example opening lines and 102 bullets; the acceptance check reads compiled, pure test, `convex-test` and native backend and refuses the five old words with its "names no tier" message (pure test). Sent back once: "the fifth tier" became the compiled tier the doc's four lack, and `design/README.md`'s tier lines followed. The Vitest project names and the design keys `tierDomain` and its siblings stay (Owner queue). +- The hosted tier. An Opus lead, 45.3 min (`aa89f34`), applied `patch-hosted-specs.diff` with sitting two's words (`f372311`; its register hunks the main thread's) and built steps 1 to 9 of section 7 of `design-hosted-tier.md` in order: the harness's backup archive names (`exportBackupArchive` and `importBackupArchive` in `harness/backup-archive.ts`), redaction at the child and admin boundary, the record's `target` apart from its tier and the acceptance filter on it, `hostedTarget` with the selection kinds (the regional URL the CLI prints accepted), the run setup and the record check, the copy's modules, the four hosted examples in one driver, `npm run test:hosted` and `.github/workflows/hosted.yml` (dispatch, and 03:00 UTC on Monday and Thursday). Locally: the planted-key test finds no form of the key in any error, output, log line or record (pure test); the undeclared table's three planting records survive a deploy of the production composition, whose schema leaves the table out (native backend, local); the four hosted examples compile, and no hosted deployment was contacted. All eight "built when" items of the design wait for the owner's first run. Its departures in items 5 and 6 went to the review as built. +- The history view on paper (4c). An Opus agent, 25.9 min (`2140eb7`), then 1.5 min on the merged branch (`bc0183e`). Its ten choices, ruled as taken: one row whatever the order across streams, each field written by one stream; `project` never null for a history projection, baselines dropped before `keyOf`; rollback refused on every path of a paused view; `HistoryRebuildTarget`, `HistorySource` and the `sourceIndex` cursor, one stream per batch, the pause length a risk; verify by repeating the backfill, one batch-size formula in events; the cross-context row narrowed to D8's parent query, with an E-9 question; the aggregate split on a key change, `entityKey` from the stream identity; the 64 KiB row budget and one `applyHistoryProjection` call per command at step 9; the pure fold in `example/domain/orderAllocationHistory.ts` with its test; the row reshaped per stock item with `OrderCancelled`, within E-46. Ten ledger findings `fixed` or `partially-fixed` on paper (`71ac95e`). The code of step 9 has no history or aggregate path (Leads 43). +- Audit and the diagnostic (4a), proof apart from fix. Phase 1, an Opus designer, 24.7 min, Spec text only (`0d2edf7`): an audit record only for an applied command and a business failure, at step 10; one diagnostic line per execution of a command entry's handler, with `relayFailure` in the outermost catch; no duration; `kind` and `code`, `eventsAppended` and `readModelRows`; a gap line in place of a counter; faults keyed by data in the fixture's `PlaceOrder`; the declaration's `diagnosticSink`. The main thread added two things at `93cadcf`: the new tier words in both Sc ALL-1 examples, and that the broken audit's error names `auditRecords` as well as `auditFault`. Phase 2: an Opus builder, 5.0 min (`8a80ae2`), with the reason E-15 asks for written into `typeDiagnosticSink` by the main thread (`12b6186`); astra at high (`job-mus3i86g-9eb1052b`, 8.7 min, `39e5472`) wrote the proof from the ruled text alone: 13 pure tests, 23 `convex-test` tests and the two Sc ALL-1 verifiers at the native backend tier, every one red before the fix. At the merge they met with one disagreement: a cut line with empty `versions` carries `versionsOmitted: 0`, as `fnDiagnosticLine` says; the code was wrong, and the integrator fixed it (`207f3b7`). +- The integration, one Opus integrator in four stages as the lanes finished: stage 1 (language, history, sweep) 3.5 min, 2a (audit) 1.8 min, 2b (the proof) 2.8 min, 2c (hosted) 10.0 min. At 2b: `npm test` 67 files, 1,010 passed, and both Sc ALL-1 verifiers passing natively. At 2c (`e97db90`), with three Spec conflicts resolved: `npm test` 69 files, 1,029 passed; `npm run test:all` once, 175 files, 1,178 passed, 2 failed and 5 skipped in 323.4 s at a load of 2.91 to 10.43, the two failures Probe 4's 1 s timeouts under load, each passing alone; `npm run acceptance` 34 passed and 4 missing (two L2-6 examples, L2-7, L2-8, no verifier yet), both Sc ALL-1 examples passed, the tree "not clean" only for the owner's untracked file. +- The review, over the branch since `840b32c`, began on partial heads so that it overlapped the hosted lane. Sol at medium read every line in two jobs, each a second time after the hosted merge: the Claude-written code (`job-mus3zboo-ae840d1a`, 2.8 and 4.9 min) and the paper (`job-mus3zbpu-c59dd380`, 4.9 and 4.5 min). An Opus agent read the GPT-written code, the sweep and the audit proof, in 10.1 min at `7603ccf`. `gpt-6-astra` at xhigh (`job-mus4kqxq-cdb3cd83`, 10.7 min, frozen `e97db90`) reviewed behavior: three findings, each also found by a second reviewer without the other (the redaction of `error.name` and of `AggregateError.errors` by sol's code read, the sweep's byte bound by the Opus read, which astra showed natively with 60 receipts of about 449 KB that a limit of 1,000 could not sweep); 17 mutations, 14 caught; both Sc ALL-1 verifiers and the undeclared-table example passing at the native backend tier. Every finding is ruled in `foldin-list.md`, F1 to F43. +- The fold-in, one Opus agent, 29.0 min, six commits `7c70057` to `804aa51`, F1 to F43: among them the sweep's refusal of a `now` later than the mutation's time and its read bounded by 8 MiB as well as 1,000 rows, `fnSweepNext` on paper, the diagnostic's cut by binary search, an async sink counted as lost, the redaction of an error's `name`, `errors` and `cause`, "backup archive" in every Spec, and the hosted examples compared over every mounted component. `npm test` 69 files, 1,046 passed; both Sc ALL-1 verifiers and the two Probe 7 files pass natively. The main thread applied F27's sentence to the local in-place example (`73ffe6d`, its native test passing), OD-155 `decided` as the owner's and OD-036 `waiting` (`c1bdf80`), and F5's ledger note. +- Sol's read of the fold-in (`job-mus5oy5q-90d6f6ac`, 4.2 min, frozen `c1bdf80`): 38 of 43 items applied as ruled and five otherwise (F5, F7, F13, F23, F38); F5 (no Spec cites a register ID) and F23 (the module keys are virtual, as a comment says) accepted, and F7, F38 and F18's test went to Leads. The main thread took one follow-up past decision 19, because a key-leak path is the hosted design's first invariant: the redaction walk returns a redacted stub past its depth bound and walks plain objects and an error's own properties, and the sweep's byte bound is asserted as the literal 8 MiB (`651fcc3`, a pure test per path). The unit stopped there. +- The whole native tier on the final code, `c1bdf80`, in a frozen copy: 175 files, 1,195 passed, 2 failed and 5 skipped in 391.7 s, at a load average of 1.92 before and 22.38 after, with this unit's agents and another project's vitest run on the machine. The two failures are Probe 4's component and nested files, the bytes-written step timing out at 1 s; rerun alone at a load of 13 to 18 they failed the same way, and at `e97db90`, at a lower load, both had passed alone (Leads 43). `npm run acceptance` on that record: 34 passed, 0 failed, 0 absent, 4 missing, "tree clean", "run failed" only for Probe 4. CI's run on the push is the proof of the whole tier. +- The ledger and the register: S4's open findings went from 15 to 4 and S5's from 19 to 16; OD-149 and OD-155 are `decided` and OD-036 `waiting`. Sitting two, round 3's unit 3d, on 2026-10-03 from about 08:00 to 09:30, on the ThinkPad, on `feature/round-2` from `840b32c`. The owner brought a table of recommended resolutions for every row and said "Please help me resolve all these forks, terminology to unlock the mini roadmap and finalization of the current feature branch." The main thread first prepared a rulings page and five lanes, and the owner stopped it: "this is excessive", then "I suggest you make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra with `gpt@libar-codex`. And rule." Decision 50 holds the method. What it did, at the tier named: @@ -42,7 +55,7 @@ Sitting two, round 3's unit 3d, on 2026-10-03 from about 08:00 to 09:30, on the - Two reads of the page, the same brief, read only: an Opus agent in 11 minutes and `gpt-6-astra` at xhigh in 5 minutes. Both agreed with every recommendation and asked for changes on four: the pure test that pins twenty-three rows (`tests/pure/acceptance.test.ts`) had to change with `acceptanceRows`; the migration ruling as recommended, a write pause of the source scope for a history-reading sweep, was new mechanism (an operator cannot close a source scope: `write-pause.sdp.md`, `src/gate/entries.ts`, held at pure test and `convex-test`), wider than the per-stream refusal E-25 rejected, and outside D9's reason for a pause, and it left reads during the sweep and a backup taken mid-migration unanswered; the 3.17 ms figure is F19's probe on the fixture and the hosted quota is still unread; `functionName` is Convex's own word for a mechanism and the field carries a declaration's or a policy's name; "scenario", "run record" and the noun "pass" needed their carve-outs. Astra's correction that the on-load migration describes the Spec and not the code holds: `src/context/adapter.ts` rejects a schema-version mismatch and migrates nothing. - The rulings. The main thread took every recommendation with the reviewers' changes, and changed the migration ruling to the narrower form: a history-reading migration is written by the sweep alone, a command or a read on a stream the sweep has not reached is refused as transient (`writePaused`), no stream type is paused, the restore's journal check inventories `awaitingBaseline` as pending work and the accepted branch restarts the sweep. The refusal field is `entry`, the name of the entry (public command, internal command, parent query or operator entry) that refused. Sixteen rows are `decided` by the owner with the rulings in `register.json`, applied by the tool from `patch-register-sitting-two.json` in the unit folder; no new fork page was written, since the owner ruled from the recommendations. - Applied at the Spec that owns each question, by the main thread: `first-experiment` (the pass rule, `acceptanceRows` with Sc ALL-1, E-17's question), `transactional-domain-platform` (OQ5), `journal`, `persistence-adapter`, `queries`, `restore` and `write-pause` (OD-041 and OD-042, the gate's migration clause cut), `d13`, `do-nothing-check` and `obligation-module` (Probe 7's questions), `d02`, `journal`, `f18` and `ordered-consumer` (D2's premise, with `constrainedBy` edges to F18), `context-component`, `d02`, `first-experiment` and `obligation-module` (OQ1), `vocabulary`, `outcome-boundary`, `queries` and `read-models` (the language), `CONTEXT.md` (twenty-nine entries added or changed), `PLAN.md` 11 and `README.md` section 8 and the E-42 row, six ledger rows, `LICENSE` and the `license` field. `check.py` OK; `npm test` 63 files, 956 tests, at the pure, `convex-test` and compiled tiers. -- Not changed: the doc, whose two hunks are the owner's (`patch-doc-sitting-two.diff` in the unit folder); the code, where the refusal field, the tier names in sixty example opening lines and the harness parser, and the harness's backup archive names now disagree with the Specs by ruling, as roadmap lane 4f; the hosted patch, applied in the hosted unit with "marker" read as planting record and a test's "pass" as test result. +- Not changed at the sitting: the doc, whose two hunks are the owner's (`patch-doc-sitting-two.diff` in the unit folder); the code, whose words lane 4f brought to the Specs in the close of round 3; the hosted patch, applied in the close with "marker" read as planting record and a test's "pass" as test result. Round 3, unit 3b, `application-platform-round-3`, on 2026-10-03 from about 03:32 to 07:00, on the ThinkPad, from `feature/round-2` at `62b8817` to `399b410`. The owner's words at the opening: "please plan and execute the scope, including validaiton of the hosted convex env. use fable 5.1 for designs of bigger pieces of solution, opus, astra and sol for execution." The hosted check at the opening was read only: the environment `hosted` shows its three variables and its secret, and `convex env list` with the owner's login reached `wandering-mule-403`; whether the key itself authenticates is the owner's one command (Owner queue). During the unit, at the integration, the owner asked whether the proof records were needed: "Tests should be all the proofs we need." and "We need to weed out all examples from the codebase and do all the suggested things." Decision 48 holds it. The main thread wrote two rulings pages, `rulings-1-round-3.md` and `rulings-2-on-the-integration.md`, and `answers-lanes.md` keeps every answer with the ruling after it. What each lane showed, at the tier named: @@ -70,38 +83,12 @@ Round 3, unit 3b, `application-platform-round-3`, on 2026-10-03 from about 03:32 - The review, `gpt-6-astra` at xhigh under an Opus lead, started on frozen `a013ba4` so that it overlapped the fold-in. First pass, 13 min 10 s: the native tier 127 passed and 21 failed (F1 to F4 and a Probe 4 timeout under load), 25 mutations with 18 caught, two text gaps (G1, G2). Its later passes, the same thread sent each head, at `314c265`, `2283083` and `de285bb`, 1 h 45 m in all over four runs: at `314c265` it raised the two findings sol's first read raised (a saved cursor's tenant read through another tenant, a partial cursor read a retired generation), at `2283083` sol's codec pair, and all four are closed at `de285bb` in both compositions, at `convex-test` and natively. Of 43 mutations 39 were caught: M17 and M20 to M23 turned caught with G3 and the fold-ins, and every cursor and codec mutation, M26 to M43, was caught; M06, M25, M31 and M32 survive (Leads 41). Eight of nine L2-5 replays were caught, and R2 survives as G4 allows. The native tier in its copy at `de285bb`: 103 files, 146 passed, 2 failed, 5 skipped, the whole run 1,102 passed, 2 failed and 5 skipped in 274.9 s, both failures Probe 4 backend timeouts that passed alone; at `314c265` 144 passed and 4 failed, each passing alone. Its one text gap, `read-models.sdp.md:99`, the main thread fixed in `399b410`: no document ID or generation number leaves the parent in a row, and a client passes a page's cursors back unread. - Three reads of the fold-in by one `gpt-6.1-sol` job at medium, each on a fresh frozen copy. At `314c265`: four findings, among them a saved cursor of another tenant that overrode the authorized tenant and a six-element cursor that read generation 2 while generation 1 was active; five tests that passed with their rule removed; every recomputed number equal (the L2-5 run's checkpoints 395 + 525 = 920 against 626 subjects, 3 registry reruns and 1 progress-row rerun against a bound of 39); seven text gaps. Ruled K1 to K4: the cursor's equality prefix compared and anything else refused with `"The cursor is not from this list"`, `getGenerations` over the configured read models, the page tests through the entries, the seam's qualification. At `2283083`: two findings with one root, `paginator`'s own codec, so a tenant `t-undefined` was refused its own continuation and `_t-undefined` read its rows. Ruled L1 to L4: the key decoded and encoded as `paginator` does, held to the helper by test. At `de285bb`: no finding on a healthy or ordinary failure path, every number equal, and four codec mutations and four text gaps left (Leads 41). The unit stopped there on decision 19. Sol's job took 41.9 minutes over four turns, a token refresh cutting the first. - The L2-5 file, native tier, production composition: 39.0 s alone at `a013ba4` (load 2.08), 72.1 s in the integrator's whole-tier run (load 1.14) and 106.6 s in its first (load 3.41); 40.6 s alone at the fold-in (load 3.97), 94.3 s and 287 s in the fold-in's two whole-tier runs under the outside load; 43.5 s and 42.3 s alone after K and L. Every time is well inside the 300 s timeout; none is a quiet-machine timing. -- The ledger and the register. Five ledger rows are `partially-fixed` with their reasons: the checkpoint on the row commands read, the tenant list, the batches bounded in documents only, and the layout's two rows of a native rollout to version 2 and a native absence, which stay with the fixture at `convex-test` by Q1. S4's open findings went from 19 to 15. OD-057 (convex), OD-071, OD-075 and OD-080 (domain) are `decided` by their advisors' checked leans, applied by the tool in `314c265`; OD-149 stays `sorted`, its lean unchecked. +- The ledger and the register. Five ledger rows are `partially-fixed` with their reasons: the checkpoint on the row commands read, the tenant list, the batches bounded in documents only, and the layout's two rows of a native rollout to version 2 and a native absence, which stay with the fixture at `convex-test` by Q1. S4's open findings went from 19 to 15. OD-057 (convex), OD-071, OD-075 and OD-080 (domain) are `decided` by their advisors' checked leans, applied by the tool in `314c265`; OD-149 stayed `sorted` then, its lean unchecked. - The sweep. The owner's words at about 04:45, after seeing records still written: "I can see we are still practicing proof records. [...] Is this needed? Tests should be all the proofs we need. If we decide to remove this, how can we actually make it happen this time?" and "We need to weed out all examples from the codebase and do all the suggested things." An Opus agent in 3.4 minutes took the pattern out of `AGENTS.md`, `SESSIONS.md` and `design/advisors/README.md` (`e377f58`) and out of the guide and the project notes. No record file was written after it. -The tooling unit, `application-platform-tooling-1`, on 2026-10-03 from about 01:50 to 03:00, on the ThinkPad, from `feature/round-2` at `8cc40e8`. The owner's words at the opening: "we will document and start designign the backed tooling updates, not do any changes in the next session. but record data for sure." So no code and no Spec changed on the branch. Each proposed Spec change is a patch in the unit folder, proven on a scratch copy and not applied, and the close commits the register, this file and the roadmap. No native backend started and no hosted deployment ran anything. The main thread wrote two rulings pages, `rulings-1-tooling-1.md` and `rulings-2-on-the-advisor.md`, and ruled on the leads' answers. What each lane showed, at the tier named: - -- Data. `gpt-6.1-sol` at medium, two turns, 9.5 minutes, sorted the eleven facts of round 2's tooling scout by the brief's rule, (a) a fact Spec bullet, (b) a new fact Spec, (c) a native harness Spec bullet, (d) a lead. Three were already carried, four went to the hosted lane, and four are (c): `functionLogOverflow`, `sourceScheduledParallelism`, `sourceHttpMutationFailure` and `sourceBackendOrigin`, Design bullets of the native harness Spec marked `[extension]`, with Probes 12 to 15 proposed as six example Specs. The reproduction of `IsolateNotClean` is (d), lead 35. Every claim rests on source and ran nowhere: the backend at `5c7cb5b`, `convex` 1.46.0 under `node_modules`, docs.convex.dev, this repository's harness. The lead checked the scout's upstream line numbers and corrected five: `knobs.rs:80-82` for the function log's 1,000 entries, `:545-549` for eight scheduled jobs, `:1080-1083` for 4 MiB a second of writes, `isolate.rs:326-335` and `local_backend/src/lib.rs:192-197` with `:208-210`. It sent sol one list of two items, Probe 13's status case to the pure tier and the `convex dev --local` example dropped. The patch, 10 files, proved on a scratch copy: 238 Specs, 373 nodes and 1,241 edges, `check.py` OK, validate 0 errors. It is not applied. Leads 30 to 38 are the lane's. -- 3b's layout. An Opus lead, reading only, wrote `layout-3b.md` from the S4 paper's package 3b and section 7. It counts thirteen registered functions per composition (eight operator entries, four batches and `getGenerations`), where the paper said eleven and the roadmap eight, both written before `fillTenants`. It lists 33 proof obligations: 1 compiled, 4 pure, 23 `convex-test` and 5 native. It plans three lanes at once (the fix by sol; the proof and the native tests by astra), then an integrator, a review and a fold-in, about three to three and a half hours. Its estimate for the native L2-5 run, not a measurement, is about 750 to 780 chained batches: 20 to 50 s on a quiet machine, and 55 to 130 s at round 2's load factor of 2.6, past the native `testTimeout` of 120 s. Sol checked it, reading only, in 4.9 minutes on a copy with one planted citation. The plant was caught, 107 citations held and 5 did not, and the lead fixed each, among them the setup count (39 native and 6 `convex-test` test files reach the first activation) and the estimate's arithmetic. Eight questions went to the owner. -- The one advisor. The owner, after the layout: "please spawn fable advisor to recommend decisions. let's keep it less noisy and use one advisor. quality, simple design, maitanable platform are guiding principles." One Fable advisor of the domain lens took three tasks, each in about five to six minutes by its files' times. The first two were the eight 3b questions (`memo-advise-3b.md`) and the hosted tier's eight decisions (`memo-advise-hosted.md`). The owner answered: "I agree with all recommendations. We have unused Convex Professional account. Let me know how to login." The third was the five register rows the check of the 3b memo sent back (`patch-register-3b.json` with its memo). Sol checked each at medium, reading only, on a copy with one planted citation, and caught every plant. On the 3b memo, 69 citations held and 11 more did not; 29 numbers and quotes held and 5 did not. Four of the memo's reasons were wrong: `documentTitle` resting on a `!` that TypeScript erases, "no tier can show" context calls where a pure test can, two ledger rows called `partially-fixed` that are `open`, and a factory that would halve the surface. Rulings 2 corrects each recommendation they touched. On the hosted memo there were 10 faults, two above the line: the usage fence and serial runs. On the register patch, 55 held and 11 did not, among them OD-075's history form, which changed its `do`, and OD-057's stale `checkedBy`. The advisor rewrote 17 lean fields, and the recheck, 2.7 minutes, found 57 held and 2 did not: OD-149's `ifWrong` and the memo's answer on whether 3b may start. -- Hosted. `gpt-6-astra` at high, 7.9 minutes, one turn, gathered the facts of a hosted deployment from `convex` 1.46.0, the backend at `5c7cb5b` and docs pages read on 2026-10-03. The lead checked all 147 citations against their sources and found 5 that misattributed a quote or a range, none a wrong fact. It added a fourth arrangement beside the scout's three, a fresh development deployment per run, which the deployments page describes. It also added GitHub's rule that a schedule runs from the default branch. The lead's own design draft and patch, written before the main thread chose to spend Fable on the decisions, are kept as `draft-hosted-opus*`: input, not a decision. Sol caught their plant and three high faults. A second Opus lead turned the advisor's memo into `design-hosted-tier.md` and `patch-hosted-specs.diff`, with the owner's fact that the plan is Professional. The design has one shared development deployment in a project and team of their own, one development deploy key scoped to fourteen actions, and one GitHub secret in an environment `hosted` that admits the default branch. Runs are serial. Four hosted examples run in one driver file (F16's retention, F12's in-place import, the production composition's acceptance, Probe 3's function calls), on a cron twice a week. Cost is in the plan's units: Probe 3's fence is 2,060 calls a run, about 18,500 a month against 25M included, and the rest is unmeasured. The 12-file patch changes five Specs, adds five example Specs, and touches the README and the register, where it adds the row OD-155. It proved on a scratch copy (237 Specs, `check.py` OK, validate 0 errors) and is not applied. Sol's check of the design caught its plant and found 14 faults; the lead fixed eleven and sent three forks, ruled in rulings 2. The usage fence stays labelled judgment, and a miss leaves OD-036 open. The retention example passes by planting and recording, which narrows E-16's fail rule. The patch's new words wait for sitting two. Every hosted statement is from source or docs. -- The setup. The owner asked: "This setup cannot be documented and executed by an agent? With gh cli, convex cli, and other tooling?", then "1. and if someting is an issue, i will jump in." An Opus agent ran design section 0 with CLI 1.46.0 at about 02:56. Two things now exist: the project `libar-platform-hosted` in the owner's one team, and its development deployment `dev/hosted`, named `wandering-mule-403`, at `https://wandering-mule-403.eu-west-1.convex.cloud`. `--expiration none` was accepted. The printed URL carries the region, so `HOSTED_DEPLOYMENT_URL` takes the printed form and not the one the design templates. The GitHub environment `hosted`, its three variables, the deploy key and the secret do not exist; they are left to the owner. The agent's own permission check refused the first `gh api` write before it ran, and the brief said to stop at the first refusal. The key was the owner's to make by hand in any case: the main thread's harness refuses to write a secret store even through an agent, a "Secret-Store Writes" denial of 2026-10-03 (decision 47). The CLI left no file. Design step 9, one deployment and no production deployment, stays a dashboard check, because CLI 1.46.0 has no `deployment list`. -- What the owner decided, as rulings 2 records it. 3b's eight questions follow the advisor, with the sharpenings and the corrections after sol's check. The `rebuildFunctions(config)` factory is not tried, because `rebuild.sdp.md:112` keeps thirteen registrations. The setup change covers the eight helpers and the test files that call `activate`. The operator term is a language row for sitting two. The hosted tier takes all eight recommendations on Professional. The setup is an agent's and the key is the owner's. The main thread ruled lane data's forks: `IsolateNotClean` stays a lead, Probes 12 to 15 keep the probe plan's numbering, and lead 33 stands. The register's seven rows are recorded at this close by the tool from the advisor's patch: OD-010 and OD-014 are `decided` by the domain advisor from their checked leans. OD-057, OD-071, OD-075 and OD-080 are `sorted` with new leans that sol's recheck held, which carry `checkedBy`. OD-149 is `sorted` with a new lean and no `checkedBy`, since its `ifWrong` did not hold. +The tooling unit, `application-platform-tooling-1`, on 2026-10-03 from about 01:50 to 03:00, on the ThinkPad, designed the backend tooling and changed no code and no Spec on the branch, by the owner's words: "we will document and start designign the backed tooling updates, not do any changes in the next session. but record data for sure." It sorted round 2's tooling scout into fact rows, native harness bullets and leads (a patch proven on a scratch copy and not applied; Leads 30 to 38); laid out 3b as three lanes and 33 proof obligations (`layout-3b.md`); asked one Fable advisor for 3b's eight questions and the hosted tier's eight decisions (decision 46; the owner: "I agree with all recommendations. We have unused Convex Professional account. Let me know how to login."); and turned the hosted memo into `design-hosted-tier.md` and `patch-hosted-specs.diff`, which the close of round 3 applied and built. An agent's setup made the project `libar-platform-hosted` and its development deployment `dev/hosted` (`wandering-mule-403`, with the regional URL as printed); the key and its secret were the owner's by hand (decision 47). Sol checked each memo, design and patch on a copy with one planted citation and caught every plant. The rulings, memos, designs, patches and checks are in the unit folder. -Round 2 of the roadmap, from the evening of 2026-10-02 to past midnight, on the ThinkPad, as nine lanes merged into `feature/round-2`, then an integration, a review and a fold-in. The owner's words at the opening: "Use a sizable chunk of the roadmap and orchestrate execution with doing minimal work in the main thread. use gpt-heavy and opus-heavy model selection and fable only for key design decisions on a larger bit of roadmap." and "Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions." During the round: "we will need a feature branch"; "i agree with the least noisy and cleanest path forward."; "good note on the load. i have moved this to machine that is not capable enough. but i will relocate to M3 mac tomorrow. hopefully we will manage today."; and on 2026-10-03, "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue", "We can discuss details before preparation of work for the next session." and "We will continue development of this project here on the thinkpad. It is not ideal, but sufficient. M3 Mac is continuting with macos native development." Decisions 40 to 45 hold what followed from them. - -The main thread ruled the forks of sitting one on the owner's word, as F1 to F12 of `rulings-1-round-2.md` in the unit folder: twelve rows decided (OD-046, OD-064 and OD-123 by one answer, OD-153, OD-068, OD-069, OD-148, OD-025, OD-150, OD-151, OD-152, OD-047) and OD-048, the licence, left the owner's. The register tool refused a session ruling on an `owner` row, the owner chose the cleanest path, and the twelve are recorded as decided by the owner, each ruling ending with both of the owner's sentences; the tool is unchanged (decision 40). OD-007 was decided by its advisor, its checked lean as the ruling. What each lane showed, at the tier named: - -- Sitting one. An Opus lead wrote the twelve answers into the rows and the forks and applied F1 to F3 at ten Specs: 100 lines is the largest order `PlaceOrder` supports, the bound its declaration carries and no platform limit; the speed targets of F2 are set for the pinned local backend before the benchmark; Sc L2-9 is read as the local observation, its concurrency half the contention run. A sol check found the twelve ruling texts and the thirteen fork answers exact and six Spec sentences that still contradicted a ruling, all corrected. Open questions fell from 144 to 138 at that lane. -- The S3 measurement (2a). `gpt-6-astra` at high, 20 minutes, one turn, on the native backend, local: ten cost cells and nine contention cells, each writing a `Measurement` record. The healthy order of 1, 10 and 100 lines read and wrote 8/6, 35/24 and 305/204 documents; a duplicate or a changed-input conflict read 5 and wrote none. In every one of the nine contention cells one command applied, N minus 1 were rejected for short stock and the engine failed none: 3, 15 and 63 executions at N of 2, 8 and 32, each loser refused once at commit and rejected on its rerun; the worst cell, 100 lines at 32, read 9,915 documents and 4,195,493 bytes for one applied order. Eight commands on unrelated stock, in one tenant or eight, applied with no rerun. The six cost targets held, the fourth as C1 counts it. A stock state that admin access set ahead of its events let an order for 3 apply against 2 received: a ledger finding for S5 (C2), no check on the command path. The journal tail read and the four-document grant read stay (C3, C4), with their costs on OD-066 and the new OD-154. The lane's own timings were taken under load, two readings 6 times apart at one line, so the comparison waited for the quiet run below. -- The S4 paper (2b). One Fable designer, 18.5 minutes, both subjects, Spec text only: a read model installed through its first rebuild, the projection chosen by the version its generation row records, progress on a `generationProgress` row only batches read, one verify pass that inserts, updates and deletes, a switch of one write, rollback as F4 rules it, `interruptGeneration`, the tenant list as the `tenants` table (E-49), the operator entry contract of F6 (E-48), the gate's scopes and refusal, and `AuditRecordInput` pinned. `design-s4-paper.md` in the unit folder has the decisions and the build cut. A sol check at medium in 8 minutes listed fifteen first findings: an Opus lead applied 11, sent 2 to the main thread and left 2 with a reason, found 3 sol did not, and cut sol's ledger draft of 37 items to 25. S4's open findings went from 31 to 19. The main thread's six rulings after the check are section 7 of the note. -- `CancelOrder` and `release` (2c). An Opus builder, then a sol read at medium with 35 mutations. Pure, `convex-test` and six native examples on the production composition: the order's `cancel` and `OrderCancelled`, the stock item's `release` and `AllocationReleased` with totals only (F9), Orders called first, the summary row replaced with `cancelled`, three new rejections. A cancel writes six documents on the native backend, and a refused release writes none. Nothing under `src/` changed. Sol's read led to twelve corrections, each new assertion shown to fail with its rule removed, and found that an `OrderPlaced` of 100 lines with 256-byte stock item IDs is 30,925 bytes, above the 16,384 bytes `append` holds a payload to. -- Probe 6 (2d). Astra at high, two turns, 40 minutes, six native examples on the fixture composition: `@convex-dev/migrations` 0.3.6 walks only the tables of the component it is defined in; it drives a context's rows only through a parent internal mutation written by hand, against a contract its README does not document, which the compiled tier needs a cast for; its status and the generation's state are two vocabularies; a batch that reads past 16 MiB fails as a whole and leaves no error on the component's row. So the rebuild drives every batch itself and uses the component for nothing (G2), F17 is probed, and the component is pinned as a dev dependency for the fixture's probe only. -- Probe 7 (2e). Astra at high, two turns, 54 minutes, five native examples with 60 bound values. Into a fresh backend `convex import --replace` restores every document with its `_id` and `_creationTime` and leaves every `_scheduled_functions` table empty, and a stored scheduler ID still validates while `system.get` returns null. In place it replaces later changes with the exported documents and keeps every scheduler row, including rows made after the export, and a kept pending mutation runs against the restored data. Scheduled arguments are bounded at 16,777,216 bytes per mutation in Convex value bytes; a single call of 8 MiB is accepted, with a warning from 80 percent of 4 MiB. A scan of `_scheduled_functions` fails at 4,092 rows on the 4,096-read limit. F12 stays documented and F16 rechecked, each with the reason; the kept schedules are a ledger finding for S5. Admin access gained `deployTemporary` and the completion's log lines, and a ZIP reader in Node replaced the `unzip` binary. -- The fact rows (3c, pulled into the round). Astra at high, 22 minutes, on the native backend under load. F18 (Probe 8): one commit timestamp per transaction in the parent and in components, larger ones later. F19 (Probes 9 and 11): 1,000 empty nested calls committed in 3.8 to 9.4 s and refused requests ended at 15.2 to 15.7 s on the system-operation budget, while a loop of computation was refused at about 1 s; 2,000 component calls committed and 4,000 were refused. F20 (Probe 10): a page of 32 rows at the 256 KiB budget read 8,390,560 bytes against the 8,388,608-byte bound, because the check runs after the row that reaches it. The "no job" assertions of Sc L2-2 and L2-3 now also read `_scheduled_functions` of the parent and both contexts, with a positive control. -- The gate (3a, pulled into the round). The Opus lane lead wrote the fix and astra at high wrote the proof apart from it from the same Spec text, in 16 minutes: 106 tests (5 compiled, 40 pure, 58 `convex-test`, 3 native), all red before the fix and all passing at the merge. The five disagreements at the merge were each a wrong test, and the fix was wrong nowhere. Built: the `tenants` table, `assertOperator`, `nextTenant`, `src/gate` and `src/audit`, step 7 before the operation is minted, step 10's audit record, and `closeGate`, `resumeGate`, `getGate` and `getGateAudit`. On the native backend the restore door's `MAINTENANCE_MODE` refuses the next command in every tenant with no deploy, a new example, and a close that races commands leaves no command committed that read the gate open: the losers rerun, 16 reruns in 8 trials. By ruling, the grant helpers read the gate too, and the trimmed operator is what is recorded. -- Polish. An Opus lead and sol at medium, 5 minutes. A native reproduction showed E2's premise: a 100-line order with 256-byte stock item IDs passed step 1, `append` threw a plain error inside Orders, and the caller got a bare technical failure with nothing stored. `PlaceOrder` and `ReceiveStock` now refuse a stock item ID above 64 bytes of UTF-8 at step 1 as `invalidInput`: 100 lines at 64 bytes make a payload of 11,725 bytes, at 128 bytes 18,125. On the native backend an order at the bound applied with 305 documents read and 204 written, and one byte past it was refused with none read. Five mutations ran red. The stale README counts were recomputed. -- The integration. An Opus integrator merged eight lane branches into `feature/round-2` on top of sitting one. The conflicts were counts, a few bullets two lanes had both written, and generated files, which one codegen after each merge left unchanged. One test disagreement between two lanes (the measurement's tables left behind against the gate's new tables) was a wrong test. At `e6e1112`, every project in one run with other lanes on the machine: 160 files, 761 tests, 754 passed, 1 failed and 6 skipped; the failure was a literal parent table name in a Probe 6 example that the gate's new table shifted, since fixed. `npm run acceptance` on that record: 31 scenarios passed, none failed or absent, 5 missing (Sc L2-5, L2-6 twice, L2-7, L2-8, which have no verifier yet). -- The review. Astra at xhigh, 13 minutes, on a frozen copy of `1b92bd8`: two findings on an ordinary path (the Probe 6 fixture callback writes a wrong status, `deploySnapshotFixture` drops the caller's signal), four surviving mutations, and two places where the S4 text did not determine the behavior (a deployment whose grants predate the tenant list, the batch size of a resumed purge). With 8 mutations and the gate and polish lanes outside its copy it was thin for xhigh, so a second pass reads the head after the fold-in. -- The second pass of the review, the same astra job at xhigh, 16 minutes, on a frozen copy of `40ea25d` with the gate, polish and fold-in in it. The whole native tier passed, 144 tests and 6 skipped. Of 34 mutations 29 were killed, and every fold-in test failed under its mutation. It found one defect on a healthy path: `getGateAudit` let a caller's `endCursor` widen a page past the cap of 100, which only admin access reaches. Beside it: a tenant ID with no bound in `insertGrant`, a sentence that did not say whether `fillTenants` reads the restore door, three measurement assertions that passed under mutation, and Probe 7's restore comparisons leaving out the four new tables. A second fold-in closed all of it as rulings J1 to J8, each new test shown to fail under its mutation (`convex-test` for the page cap and the tenant ID bound, a pure test for a decomposed stock item ID, native for the measurement and Probe 7). A sol read of that fold-in at medium, 7 minutes, found nothing on a healthy or ordinary failure path and recomputed every number with no difference, so the round stopped on tactical decision 19. -- The fold-in. An Opus agent with no GPT job, because the rulings named every change: the caller's signal reaches the deploy child (pure test), three `CancelOrder` tests at `convex-test` over two stock items, a second tenant and two orders, each shown to fail under its mutation, and the Spec text of `fillTenants`, the refusal of `startGeneration` on an empty tenant list beside a grant, and a resumed pass keeping its batch size. At `40ea25d`: typecheck, lint and format pass, `npm test` 59 files and 615 tests, the six native files that reach `deployTemporary` pass, `check.py` OK. -- The timings on the quiet ThinkPad, at `8421d6b` with no other job running (ruling J7; the logs and pins are `timings-place-order-quiet.log` and `timings-f19-quiet.log` in the unit folder), on the native backend, local. `PlaceOrder`'s latency, median and 95th percentile: 20.3 and 21.3 ms at 1 line, 35.1 and 52.4 ms at 10, 166.3 and 178.3 ms at 100. Throughput: 85.9 ten-line orders a second from 8 callers over 10 seconds, 864 applied, no rerun, beside the local write cap of 4 MiB a second. Every F2 target held, the contention target included. F19's probe, 100 reads a sample: a median of 7.2 ms through a helper, 208.6 nested and 324.6 through a component, 3.17 ms per component call over a helper call. The Specs carry them: `observedLatency` and `observedThroughput` in the first-experiment Spec, F19's `nativeMeasurements`. - -The branches. Each lane worked in `~/dev-libar/wt/` on a branch of its own, from `main` at `592f1dc` for the first seven and from `feature/round-2` or `unit/s4-paper` for gate and polish, and the integrator merged each into `feature/round-2` in `~/dev-libar/wt/round-2`. The branch is pushed. The owner's words: "we will need a feature branch", then "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue". So no pull request goes to `main` until the owner has reviewed `feature/round-2`. +Round 2 of the roadmap, `application-platform-round-2`, from the evening of 2026-10-02 to past midnight, on the ThinkPad: nine lanes merged into `feature/round-2`, then an integration, a review and a fold-in. Decisions 40 to 45 hold the owner's words of the round. The main thread ruled sitting one's forks on the owner's word, twelve rows recorded as the owner's and OD-048 left open (decision 40). What the lanes showed, at the tier named: the S3 measurement on the native backend, local, in which every contention cell applied one command and rejected the rest for short stock, and the six cost targets held; the S4 paper by one Fable designer (installation through the first rebuild, the tenant list, the operator entry contract; S4's open findings 31 to 19); `CancelOrder` and `release` with totals only; Probe 6, after which the rebuild drives every batch itself because `@convex-dev/migrations` drives a context's rows only through a hand-written parent mutation; Probe 7, a fresh import keeping no schedule and an import in place keeping every one, with 16 MiB of scheduled arguments per mutation; the fact rows F18 to F20; the gate, 106 proof tests written apart from the fix and all passing at the merge with the fix wrong nowhere; the stock item ID bound of 64 bytes. Astra's review at xhigh, in two passes, and one sol read of the fold-in closed the round on decision 19. On the quiet ThinkPad at `8421d6b`, native backend, local: `PlaceOrder`'s median latency 20.3, 35.1 and 166.3 ms at 1, 10 and 100 lines, 85.9 ten-line orders a second from 8 callers, every F2 target held; F19's 3.17 ms per component call over a helper call. The Specs carry the numbers, and the rulings, answers and timing logs are in the unit folder. Round 1 of the roadmap, on 2026-10-02, in one evening on the second machine, as four lanes on their own branches, merged through pull requests #6 to #9. No Fable agent ran: the owner said the same day that the weekly Fable limit is better spent on implementation, and decisions 36 to 38 hold the rulings. What the round showed, each at the tier named: @@ -174,21 +161,21 @@ What passed, and at which tier, with backend release `precompiled-2026-09-28-5c7 Every Layer 1 scenario runs on the fixture composition. The Layer 2 scenarios run on the production composition, except the one that needs a fault. -What the code has and what it does not have yet, after round 3's unit 3b. The Specs pin all of it and say nothing about which part is built: the owner's direction of 2026-10-01 is that a Spec carries no temporal information. +What the code has and what it does not have yet, at the close of round 3. The Specs pin all of it and say nothing about which part is built: the owner's direction of 2026-10-01 is that a Spec carries no temporal information. -- Pipeline: both entries, steps 1 to 6 with the ID length bound, the gate read of step 7 through `assertWritable` before the `OperationRef` is minted, step 8, step 9 with the `writes` check, each writable generation written with the projection its row's version selects, the row budget and the row count, the receipt insert of step 10 with the audit record of a declaration that sets `audit`, and the return of step 11. Not yet: the diagnostic. -- Declaration: `writes` is required, `readModels` and `audit` are optional. No `irreversible` field yet, and `retention.afterExpiry` is `"delete"` only. There is no `commandRegistry` and no registration check: the Specs dropped them, and the compiler over the generated api is the check of Sc L2-4. +- Pipeline: both entries, steps 1 to 6 with the ID length bound, the gate read of step 7 through `assertWritable` before the `OperationRef` is minted, step 8, step 9 with the `writes` check, each writable generation written with the projection its row's version selects, the row budget and the row count, and the return of step 11. Step 10 inserts the receipt, then, for a declaration that sets `audit`, the audit record of an applied command or a business failure only; a failed audit insert throws, and the command rolls back as a technical failure with nothing stored. The diagnostic (`src/operations/diagnostic.ts`): one line per execution of a command entry's handler, written at the end through the declaration's `diagnosticSink` (`consoleSink` when it names none), with `relayFailure` in the outermost catch, at most 4,096 bytes with `versions` cut by a binary search for the longest prefix that fits, no duration, and one gap line, never a throw, when the sink throws or returns a thenable. Step 9 has no history or aggregate path. +- Declaration: `writes` is required, `readModels`, `audit` and `diagnosticSink` are optional. No `irreversible` field yet, and `retention.afterExpiry` is `"delete"` only. There is no `commandRegistry` and no registration check: the Specs dropped them, and the compiler over the generated api is the check of Sc L2-4. - The gate and the audit records: `src/gate` and `src/audit`. The `maintenanceGates`, `operatorAudit` and `auditRecords` tables, `gateAllows` reading `MAINTENANCE_MODE` before the gate document, `assertWritable`, `scopesOfUseCase`, `closeScope` and `resumeScope` with their audit record in the same mutation, `writeAudit` and `writeOperatorAudit` with no catch, and the four operator entries `closeGate`, `resumeGate`, `getGate` and `getGateAudit` that both compositions re-export. Not yet: a source scope closed by `startGeneration`, the restore's entries, the audit queries by tenant, a typed `env`. - Authority: `establishActor`, `authorize`, the grants table, the `tenants` table, and `insertGrant`, which inserts a tenant's row with its first grant, `revokeGrant` and `nextTenant`; both grant helpers read the gate first. `assertOperator` checks the stated operator of an operator entry and returns its trimmed text. The library registers no grant function; each composition registers its own two internal grant mutations. `fillTenants` and `fillTenantsBatch` fill the tenant list from the grants, with their progress on the `tenantFill` row. -- Read models: the per-entity form only, written from the DTOs of a command's stream entries. A declaration carries `projections` by version, latest first, and `projectionOf` picks the one a generation's row records. The slimmer `generations` row, `generationProgress` with its `pass`, the `projectionMarkers` table, the three roles of a written generation, and the backfill mode with its `null` rule. The rebuild is built: `src/read-model/rebuild.ts` with one helper per entry over `RebuildConfig`, `batchSizeFor` and `resumeChain`, and in each composition a `rebuild` module of thirteen functions (`startGeneration`, `interruptGeneration`, `resumeGeneration`, `switchGeneration`, `rollbackGeneration`, `abortGeneration`, `purgeGeneration`, `fillTenants`, the backfill, verify, purge and fill batches, `getGenerations`). The first activation is gone: every read model, in every setup, is installed through its first rebuild, and the renamed example `first-rebuild-installs-read-model` runs on the fixture. A parent read-model list pages with `paginator` and moves the client's cursors into the active generation with `pageInGeneration`, refusing a cursor of another list. Not yet: the paused path (`pauseRequired`, source scopes, history batches), the aggregate rebuild and `applyAggregate`, `applyHistoryProjection`, a stall check, a purge that starts by itself. -- Receipts: the key, the fingerprint, lookup, classification and insert. An expired receipt is treated as absent and deleted in the same mutation. Not yet: tombstones and the sweep. -- Outcome boundary: the wire shapes, `reject`, `refuseTransient`, `normalizeThrown` with the closed code list, `classifyThrown`. Not yet: the conditional dispatcher and `carryOrReject`, which has no caller. +- Read models: the per-entity form only, written from the DTOs of a command's stream entries. A declaration carries `projections` by version, latest first, and `projectionOf` picks the one a generation's row records. The slimmer `generations` row, `generationProgress` with its `pass`, the `projectionMarkers` table, the three roles of a written generation, and the backfill mode with its `null` rule. The rebuild is built: `src/read-model/rebuild.ts` with one helper per entry over `RebuildConfig`, `batchSizeFor` and `resumeChain`, and in each composition a `rebuild` module of thirteen functions. Every read model, in every setup, is installed through its first rebuild. A parent read-model list pages with `paginator` and moves the client's cursors into the active generation with `pageInGeneration`, refusing a cursor of another list. The pure fold of the order allocation history is `example/domain/orderAllocationHistory.ts`, with its test and nothing that calls it. Not yet: the paused path (`pauseRequired`, source scopes, history batches), the aggregate rebuild and `applyAggregate`, `applyHistoryProjection`, a stall check, a purge that starts by itself. +- Receipts: the key, the fingerprint, lookup, classification and insert. An expired receipt is treated as absent and deleted in the same mutation. The sweep: `receipts.sweep`, one internal mutation for one tenant over `by_tenant_expiry` up to a `now` no later than the mutation's own time, ending at the caller's limit, 1,000 rows or 8 MiB read (`boundedPage` with `limitSweepBytes`, which is `limitListBytes`), whichever comes first, deleting each expired row it read, an expired tombstone included, and answering `deleted`, `compacted` (0) and `more`; and `receipts.sweepNext`, one such run for the next tenant of the tenant list after the caller's `after`. Both compositions register both; nothing schedules them. Not yet: tombstones and compaction. +- Outcome boundary: the wire shapes, `reject`, `refuseTransient`, `normalizeThrown` with the closed code list, `classifyThrown`. A refusal names the entry that refused in `entry`; `commandType` stays on receipts, declarations, the idempotency key, audit records, the envelope's cause and the dispatcher. Not yet: the conditional dispatcher and `carryOrReject`, which has no caller. - Context: operations, `get` and `list` with `includeDeleted` for a trusted caller. Not yet: `history`, `rebuild`, `byOperation` and `maintenance.writeBaseline`. -- Queries: a parent query authorizes through `authorizeQuery`, and its refusal is the rejection wire shape with the query's name in `commandType`. A client pages a list by a cursor pair, and no client hook crosses the component boundary. -- Journal and adapter: the single mapping only. Not yet: baselines and migrations (step 3a, `migrateToSchemaVersion`, `defineMaintenance`, the driver), `rebuildStream`, the derived mapping and `streamParts`. A stream row behind the registration's `stateSchemaVersion` is a plain error for now. +- Queries: a parent query authorizes through `authorizeQuery`, and its refusal is the rejection wire shape with the query's name in `entry`. A client pages a list by a cursor pair, and no client hook crosses the component boundary. +- Journal and adapter: the single mapping only. Not yet: baselines and migrations (step 3a, `migrateToSchemaVersion`, `defineMaintenance`, the driver), `rebuildStream`, the derived mapping and `streamParts`. A stream row behind the registration's `stateSchemaVersion` is a plain error for now (Leads 42). - The example domain: `PlaceOrder`, `ReceiveStock` and `CancelOrder`, the order's `cancel` and the stock item's `release`, `OrderCancelled` and `AllocationReleased`, with a stock item ID bound of 64 bytes on `PlaceOrder`'s lines and `ReceiveStock`'s items. `getOrderWithStock` and the parent queries over Inventory are not built, because no Spec pins their shapes; `inventory.read` is seeded and nothing asks for it yet. `installedLayers` of the production composition is `[]`, because the Spec is silent. -- Harness: admin access reads `_scheduled_functions`, exports and imports a snapshot, deploys a temporary copy of a composition with `deployTemporary` and hands back a completion's log lines; a ZIP reader in Node reads the export. -- Not yet, beyond the per-entity rebuild: the history view (a unit of round 5 by F10, its rules first in 4c), baselines and migration, the restore procedure with its `finishRestoreRun`, the receipts sweep and the diagnostic. +- Harness: admin access reads `_scheduled_functions`, exports and imports a backup archive (`exportBackupArchive` and `importBackupArchive` in `harness/backup-archive.ts`), deploys a temporary copy of a composition and hands back a completion's log lines; a ZIP reader in Node reads the archive. Every form of a key is redacted at the child and admin boundary, in output, an abort reason and an error's `name`, `errors`, `cause` and own properties, with a redacted stub past the walk's depth bound; the HTTP and socket clients log through a redacting logger. A run record names its `target`, the local backend or a hosted deployment, apart from its tier, and the acceptance check reads the four tier names and only a record of the local backend. The hosted target is built and has never run on a hosted deployment: `hostedTarget`, the selection kinds, `harness/hosted.ts`, `harness/hosted-run.ts`, `scripts/hosted-record-check.mjs`, `vitest.hosted.config.ts` with the project `hosted`, `tests/hosted/` and `.github/workflows/hosted.yml`. +- Not yet: the history view's build (step 9 has no history or aggregate path; 5d), baselines and migration (5a), the restore procedure with its `finishRestoreRun` (5b), the adopter-facing cut (5c), Layers 3 to 6. What changed in the Specs during S1, and why: @@ -219,26 +206,24 @@ Review round 3 stands as it was for the rest of the corpus: no lens approved. Th | Slice | Builds | Open findings | Blocker, major, minor | |---|---|---|---| | S3 | Measurement: 1, 10 and 100 lines, without and with contention. Repeated after S4 and S5 | 1 | 0, 0, 1 | -| S4 | Generations and rebuild, the history view and the write pause, with the tenant list, operator entry points and gate audit the rebuild uses. The paper, the gate, Probe 6 and the per-entity online rebuild are done | 15 | 0, 6, 9 | -| S5 | Baseline migration, restore, the rest of operators and audit, diagnostics, the receipts sweep, maintenance jobs. Probe 7's local part is done | 22 | 0, 12, 10 | -| L3 | Layers 3 to 6, written from what the experiment shows | 31 | 0, 12, 19 | -| P | Polish with no effect on what is built: citations, markers, stale sentences, README | 30 | 0, 0, 30 | +| S4 | Generations and rebuild, the history view and the write pause, with the tenant list, operator entry points and gate audit the rebuild uses. The paper, the gate, Probe 6, the per-entity online rebuild and the history view on paper are done | 4 | 0, 0, 4 | +| S5 | Baseline migration, restore, the rest of operators and audit, diagnostics, the receipts sweep, maintenance jobs. Probe 7's local part, command audit, the diagnostic and the sweep are done | 16 | 0, 9, 7 | +| L3 | Layers 3 to 6, written from what the experiment shows | 30 | 0, 12, 18 | +| P | Polish with no effect on what is built: citations, markers, stale sentences, README | 29 | 0, 0, 29 | -Thirty more findings are `partially-fixed`, each waiting for its slice. Remove an unsupported generic promise before designing a helper no view needs. +Thirty-eight more findings are `partially-fixed`, each waiting for its slice. Remove an unsupported generic promise before designing a helper no view needs. ## Next unit -`ROADMAP.md` is the order, ruled as cut by decision 36. Rounds 1 and 2 are done, and so are 3b and sitting two, round 3's first and third items. The next unit is the hosted tier, round 3's item 2, a build unit commissioned from `design-hosted-tier.md` in the tooling unit's folder: its build cut, its order and its "built when" list. The owner's words of 2026-10-03: "Local backend should be primary, IMO. But happy to take the hosted as default if easier for start." So the local backend stays the primary target, and the hosted deployment is a target of the native tier on a schedule, never in `test:all`. Before it starts: +`ROADMAP.md` is the order, ruled as cut by decision 36. Rounds 1 and 2 are done, and so is round 3's mini roadmap but for the hosted tier's first runs. The next unit is the owner's: the review and merge of the pull request of `feature/round-2` into `main`, which the main thread opens at this close. After the merge: -- Done: the setup, which made the project `libar-platform-hosted` and its development deployment `dev/hosted` (`wandering-mule-403`). -- Done by the owner at the owner's own prompt on 2026-10-03: the GitHub environment `hosted` with its `main` branch policy and the three variables (`HOSTED_DEPLOYMENT_URL` as the printed URL with the region, `HOSTED_PLAN` as `Professional`), and the secret `HOSTED_DEPLOY_KEY` in that environment. -- Done on 2026-10-03, at the owner's request, as the owner's inspection of the dashboard and GitHub recorded it (`tmp-convex-findings.txt`, the owner's file): the first key, `hosted-ci`, carried 25 allowed actions where the design calls for 14, so it was replaced by `hosted-ci-scoped` with exactly the 14, the secret was replaced through stdin (GitHub's update time 2026-10-03T06:31:36Z), `HOSTED_DEPLOY_KEY_NAME` is `hosted-ci-scoped`, and `hosted-ci` was revoked; one key remains. The key-only check passed with CLI 1.46.0: `convex env list --names-only` in a fresh temporary home with no stored login. The CLI refuses an empty directory, so the check needs a minimal `package.json` naming `convex` 1.46.0 beside it, with no install; the Owner queue's procedure below says so now. -- Still the owner's: the dashboard check. `clean-capybara-16` is a production deployment, reference `production`, not a development one as this file said before; never deployed, no tables, no keys, created by the project's default. The design wants `dev/hosted` only, so the owner deletes it if nothing uses it. Deletion is permanent and an agent does not do it. -- First in the unit: `patch-hosted-specs.diff` applied, with sitting two's two word changes: its "marker" is a planting record, and a test's "pass" is a test result. The regional URL stays the printed form, and the selector's check that key and URL name one deployment must accept it (lead 39). +- The hosted tier's first runs, the owner's steps 10 and 11 of `design-hosted-tier.md`: first from the owner's machine with `npm run test:hosted`, then the workflow on `main`, which the `hosted` environment admits, since GitHub runs a schedule only from the default branch. They settle the design's eight "built when" items, OD-036's count, and Leads 39 and 43's assumptions about a hosted deployment. +- The dashboard check of `clean-capybara-16`: a production deployment, reference `production`, never deployed, with no tables and no keys, made by the project's default. The design wants `dev/hosted` only, so the owner deletes it if nothing uses it. Deletion is permanent and an agent does not do it. +- Then the roadmap's remaining units, 4e (`ready` by family, the owner's), 5a, 5b, 5c and 5d, discussed with the owner before they are prepared (decision 42). -Its first runs are the owner's from this machine. The four hosted examples run by the workflow once it is on `main`, because GitHub runs a schedule from the default branch, and the `hosted` environment admits `main` only, so the first hosted CI run is a step after the reviewed branch reaches `main`. +The setup is done: the project `libar-platform-hosted` and its development deployment `dev/hosted` (`wandering-mule-403`, `HOSTED_DEPLOYMENT_URL` in the regional form the CLI printed), the GitHub environment `hosted` with its `main` branch policy and its three variables (`HOSTED_PLAN` is `Professional`, `HOSTED_DEPLOY_KEY_NAME` is `hosted-ci-scoped`), and the key `hosted-ci-scoped` with exactly the fourteen actions as the secret `HOSTED_DEPLOY_KEY`, which passed the key-only check (Owner queue); the first key, with 25 actions, is revoked. The local backend stays primary, by the owner's words of 2026-10-03: "Local backend should be primary, IMO. But happy to take the hosted as default if easier for start." -`slice` S3 is round 1's attribution and round 2's measurement, the quiet timings included. S4 is round 2's paper and gate, round 3's online rebuild, and the builds of round 4. S5 is round 5. The paper decisions S4's rebuild needs are taken; those of the history view and of S5 are listed below. +`slice` S3 is round 1's attribution and round 2's measurement, the quiet timings included. S4 is round 2's paper and gate, round 3's online rebuild, and the builds and paper of round 4. S5 is round 5. The paper decisions S4's rebuild and the history view need are taken; those of S5 are listed below. How S2 was run, for the next slice. Tactical decisions 25 to 27 hold the rules. Two Fable designers with disjoint subjects took about twenty minutes. One workflow run of thirteen Opus agents then wrote the Specs, built four packages in sequence, bound the scenarios in four worktrees and integrated, in about an hour and a half. The main thread stopped and resumed that run once, at a seam between two packages, to add corrections: finished agents came back from the cache. The scripts, the rulings and the reports are in the S2 folder the project notes name. @@ -246,13 +231,13 @@ What S2 left, as it stands now: - The measured read count of `PlaceOrder`, 3N + 5, has every term attributed (round 1) and stays the cost (round 2, C3). - Sc L2-9's native observation of the deployed function list stays deferred: OD-021's lean takes the committed generated api as the pinned list. Its concurrency half is bound, the contention run. -- A binding admits only the per-entity read model. The build that takes Sc L2-6 says how a history and an aggregate read model are bound and installed. +- On paper a binding takes the history form (4c) and the aggregate form (F9 of the close of round 3); the code admits only the per-entity read model until 5d builds the history view and OD-075's view the aggregate. - The first activation is gone: 3b installs every read model through its first rebuild, which reads the source events that already exist. Decisions each later slice needs on paper before its code: -- S4, the history view (roadmap 4c): order-independent cross-stream folds and whether deletion keeps fold memory (`r3-architecture-history-fold-order-and-deletion`); the paused path's cursor across sources, its verify pass and its batch bound; the L2-6 interrupt example's "can roll back", which the rebuild Spec now answers with a refusal. The rest of S4's paper was taken in round 2: `design-s4-paper.md` sections 1 and 7. -- S5: the history-reading sweep's transient refusal and the restore check's pending-work inventory, ruled at sitting two (OD-041) and designed with the baseline unit; which diagnostic outcomes are emitted and where a duration comes from; what a restore in place does with the schedules it keeps (`probe-7-architecture-restore-kept-schedules-not-reconciled`); how the restore check detects a stream state that is not the fold of its events (`s3-measurement-architecture-current-state-not-checked-against-event-fold`); the read-only comparison of the restore's read-model check (`s4-paper-sdp-restore-read-model-check-compare-mode-undefined`). +- S4, the history view: taken on paper in 4c (`design/specs/application/**`, ten ledger findings); what is left is the build, 5d, with Leads 43's step 9 gap. The rest of S4's paper was taken in round 2: `design-s4-paper.md` sections 1 and 7. +- S5: the history-reading sweep's transient refusal and the restore check's pending-work inventory, ruled at sitting two (OD-041) and designed with the baseline unit, including a read for the journal check that `list` does not refuse with `writePaused` (restore's open question from F5); what a restore in place does with the schedules it keeps (`probe-7-architecture-restore-kept-schedules-not-reconciled`); how the restore check detects a stream state that is not the fold of its events (`s3-measurement-architecture-current-state-not-checked-against-event-fold`); the read-only comparison of the restore's read-model check (`s4-paper-sdp-restore-read-model-check-compare-mode-undefined`). What the platform review of 2026-10-02 asks of the later units. The ledger owns each finding's counterexample and what its build must show. This is the order and the guidance that fits no single finding. @@ -271,7 +256,7 @@ What the platform review of 2026-10-02 asks of the later units. The ledger owns ## Tactical decisions taken on 2026-10-01 -The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words, 46 and 47 during the tooling unit on 2026-10-03, 48 and 49 during round 3's unit 3b the same day, and 50 and 51 at sitting two the same day. Any of them can be reopened. +The owner accepted the first nine on the session's recommendation. The session took 10 to 19 during S0, 20 to 23 during S1, 25 to 27 during S2 and 28 to 31 during the platform review unit, on 2026-10-02. Decisions 32 and 33 are the owner's words of the same day. Decisions 40 to 45 were taken during round 2, on 2026-10-02 and 2026-10-03, most of them on the owner's words, 46 and 47 during the tooling unit on 2026-10-03, 48 and 49 during round 3's unit 3b the same day, 50 and 51 at sitting two the same day, and 52 at the close of round 3 the same day. Any of them can be reopened. 1. Slices replace the eight fix units of the previous plan. 2. A slice is built on Specs at `defined` with no open blocker inside the slice. The owner states `ready` after the slice passes. @@ -314,7 +299,7 @@ The owner accepted the first nine on the session's recommendation. The session t 39. A lane's worktree. Each lane of a round is a git worktree under `~/dev-libar/wt/` on its own branch from `main`, with `node_modules` a link to the main tree's and the backend binary named by `CONVEX_BACKEND_BINARY`, and no `.cache` link, because `.gitignore` ignores the directory and not a link to it. Every brief says: never run `npm ci` or `npm install` in a worktree. A link is never created over an existing link. The main thread merges `main` into a lane by hand where two lanes touched one line, and runs the gates before the pull request. 40. The owner delegated the forks of sitting one to the main thread on 2026-10-02 ("Decide on the forks in the roadmap. Previous thread orchestrated many, many agents and collected many valuable data points, but failed to converge to a roadmap and make tactical decisions.") and, when the register tool refused a session ruling on an `owner` row, chose the cleanest path: "i agree with the least noisy and cleanest path forward." So the twelve rows are recorded as the owner's, each ruling ending with both sentences, the tool is unchanged, and OD-048 stays open. 41. A round's lane is owned by an Opus lane lead from the brief to the last commit. It writes the GPT job's brief, starts the job with the companion from Bash, reads every line the job wrote, fixes what sits below the line itself, sends one numbered list of what sits above it back to the same job, and answers the main thread in fifteen lines. Since decision 48 the lead writes no record: the answer is the record, and the job's report stays in its job directory. The main thread rules on the answers and merges. Nine lanes ran so in round 2 and three in 3b; the project notes hold the numbers. -42. The owner's words of 2026-10-02 and 2026-10-03: "we will need a feature branch" and "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue". So a lane branches from `feature/round-2` and merges back into it, and nothing goes to `main` until the owner has reviewed the branch. Decisions 32 and 39 hold for the rest: a review before `main`, and a worktree per lane. And: "We can discuss details before preparation of work for the next session." +42. The owner's words of 2026-10-02 and 2026-10-03: "we will need a feature branch" and "I would prefer continuing the roamap in the same feature branch, to be able to review the work in detail, instead of merging branches to continue". So a lane branches from `feature/round-2` and merges back into it, and nothing goes to `main` until the owner has reviewed the branch. Decisions 32 and 39 hold for the rest: a review before `main`, and a worktree per lane. And: "We can discuss details before preparation of work for the next session." At the close of round 3 the owner's words "forks have been resolved and mini roadmap unlocked. please complete the work to wrap up the feature branch and PR. use Opus, GPT 6 Astra and 6.1 Sol and orchestrate from the main thread" end the branch: it goes to `main` as one pull request, which the main thread opens at that close and the owner reviews and merges. 43. One Fable designer took the whole S4 paper, both subjects, because the gate, the rebuild and the operator entries share surfaces, and the owner said "fable only for key design decisions on a larger bit of roadmap". Opus leads, sol and astra did the rest. Fable was spent on the main thread and that one designer. 44. The ThinkPad (Fedora, an Intel i5 with 16 GB) is the development machine for now. The owner's words of 2026-10-03: "We will continue development of this project here on the thinkpad. It is not ideal, but sufficient. M3 Mac is continuting with macos native development." Seven lanes at once drove its load average to 13 to 18 and made native tests time out. So at most three or four lanes that start native backends run at once, a test that fails on a timeout is rerun once alone, and a latency or throughput timing is taken while nothing else runs. 45. An integrator never uses `pkill` on vitest: `pkill -f "vitest run"` matched every lane's run once and interrupted another lane's native file. A run is stopped by its own process ID. @@ -324,31 +309,35 @@ The owner accepted the first nine on the session's recommendation. The session t 49. The order of the work after the lanes. The integrator starts in two stages while a lane still runs: it merges the lanes that are done, rules their failures, and takes the last lane's head by message. The review starts on the merged head before the fold-in, so that its mutations and its native runs overlap the fold-in, and the same job is sent each fold-in head by message and reads the diff from the merged head to it. 50. An owner sitting is one recommendations page, two reads, then the ruling. The owner's words of 2026-10-03, after stopping a rulings page with five lanes ("this is excessive"): "I suggest you make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra with `gpt@libar-codex`. And rule." So the main thread writes one recommendation per row in the unit folder, one Opus agent and one `gpt-6-astra` job at xhigh read it on the same read-only brief, and the main thread rules and applies in its own thread. No fork page is written for a row the owner rules this way; the ruling lives in the register row, and the recommendations and both reads in the unit folder. The rows are recorded as the owner's, as decision 40 did. 51. A paper sitting renames nothing in the code. The words sitting two ruled leave the refusal field, the tier names in the examples' opening lines and the harness parser, and the harness's backup archive names disagreeing with the Specs on purpose, as roadmap lane 4f, so that the branch the owner reviews stays green and small. +52. How the close of round 3 ran, at the owner's word in 42. The main thread was Opus 5.5 and no Fable agent ran: Opus agents designed on paper where a Fable designer did before (4c, and 4a's Spec text), and the main thread ruled every sentence they wrote. 4a ran proof apart from fix (decision 29): an Opus builder and astra wrote from the ruled text alone, and the integrator merged them. The integration ran in four stages as the lanes finished (decision 49). The review over the branch since `840b32c` was four line reads by sol and one behavior review by astra at xhigh, with an Opus agent reading the GPT-written code and sol the Claude-written code (decision 30). One fold-in and one read of it followed, then the stop (decision 19). ## Owner queue Every open decision is a row of `design/decisions/register.json`, and `python3 design/tools/decisions.py` filters it: `--owner` lists what waits for the owner, forks first, `--owner --before S4` what the owner must decide before S4, `--decided --by advisor --since ` what agents decided. The register at the close: ``` -total: 154 -status: unsorted=0, sorted=96, waiting=22, decided=21, folded=15 -class: tactical=3, delegated=86, owner=50, null=15 -advisor: convex=36, domain=45, operator=34, product=39 +total: 155 +status: unsorted=0, sorted=80, waiting=21, decided=39, folded=15 +class: tactical=3, delegated=86, owner=51, null=15 +advisor: convex=37, domain=45, operator=34, product=39 ``` The key check is done ("Next unit"). For the record, the procedure that worked with CLI 1.46.0, which refuses a directory with no `package.json`: in a fresh temporary directory, write `{"name":"libar-hosted-key-check","private":true,"dependencies":{"convex":"1.46.0"}}` as `package.json`, install nothing, paste the key at a silent `read -rs CONVEX_DEPLOY_KEY`, export it, run the repository's `node_modules/.bin/convex env list --names-only` under a fresh temporary `HOME`, then unset the variable. An empty list with no error means the key authenticates. -What waits for the owner after sitting two: +What waits for the owner at the close of round 3: -- The doc's two hunks, from OD-008 and OD-044, in `patch-doc-sitting-two.diff` of the unit folder: `git apply ~/dev-libar-supporting-context/calibration/application-platform-sitting-two/patch-doc-sitting-two.diff` from the repository root, then a commit. Only the owner edits the doc. -- The dashboard deletion of `clean-capybara-16` ("Next unit"). -- The next language rows, words the ruling did not cover, which keep the code's names until the owner rules: interrupt, park, stall, batch chain, fill, the aggregate's marker, history view and history projection, tombstone, drill, door, gate, audit record, diagnostic, adopter, package, activation record. +- The pull request of `feature/round-2` into `main`, which the main thread opens at this close: the owner reviews and merges it. +- The doc's two hunks, from OD-008 and OD-044, in `patch-doc-sitting-two.diff` of sitting two's folder: `git apply ~/dev-libar-supporting-context/calibration/application-platform-sitting-two/patch-doc-sitting-two.diff` from the repository root, then a commit. Only the owner edits the doc. +- The hosted tier's first run, from the owner's machine and then by the workflow on `main`, and the dashboard check of `clean-capybara-16` ("Next unit"). OD-036, Probe 3's quota half, is `waiting` for that run. +- The untracked `tmp-convex-findings.txt` at the repository root, the owner's notes of the hosted key rotation, makes every local run record "not clean" until the owner moves it. +- OD-135 and OD-095 wait for their advisors when the owner next asks for a sort: OD-135's lean held 9 of 19 claims and needs its advisor's correction, and OD-095's held 6 of 6 but is a `do` alone and needs its lean in full. +- The language rows still open, which keep the code's names until the owner rules: interrupt, park, stall, batch chain, fill, the aggregate's marker, history view, history projection, tombstone, drill, door, gate, audit record, diagnostic record, diagnostic gap, adopter, package, activation record; and from this unit Probe 3's "set of calls", the one "preview deployment", the Vitest project names (`types`, `pure`, `simulator`, `native` and now `hosted`) beside the four tier names, and the design keys `tierDomain` and its siblings, which keep their IDs. -Decided since the last close: sixteen `owner` rows at sitting two, recorded as the owner's with the rulings in the rows (decision 50): OD-006, OD-008, OD-009, OD-017, OD-022, OD-029, OD-041, OD-042, OD-044, OD-048, OD-063, OD-110, OD-114, OD-138, OD-139 and OD-140. `decisions.py --decided --by owner --since 2026-10-03` lists them. Before that, in 3b and the tooling unit, OD-057 by the convex advisor and OD-010, OD-014, OD-071, OD-075 and OD-080 by the domain advisor, each with its checked lean as the ruling; OD-149 stays `sorted` with its `checkedBy` null, because its `ifWrong` did not hold. At sitting one, twelve `owner` rows were recorded as the owner's (decision 40), and OD-007 was decided by the convex advisor. OD-154, the grant read, is a `delegated` row with its measured cost. The hosted patch, when it is applied, adds OD-155, the hosted tier's money, as an `owner` row, and OD-036, Probe 3's quota half, takes its lean from that patch. +Decided since the last close: OD-149, the tenant created behind a pass's cursor, by its operator advisor's checked lean (`19738b4`); and OD-155, the hosted tier's money, as the owner's, from the owner's words of the tooling unit on the advisor's eight hosted recommendations, applied by the main thread's register patch (`c1bdf80`), which also set OD-036 `waiting`. `decisions.py --decided --since 2026-10-03` lists them with sitting two's sixteen `owner` rows (decision 50) and the advisor rows of 3b and the tooling unit (OD-057 by the convex advisor; OD-010, OD-014, OD-071, OD-075 and OD-080 by the domain advisor). At sitting one twelve `owner` rows were recorded as the owner's (decision 40) and OD-007 was decided by the convex advisor. OD-154, the grant read, is a `delegated` row with its measured cost. OD-093's provisional reading still quotes the old E-9 text, a field the tool does not patch (Leads 43). -`delegated` rows that wait for the next sort of their lens, each with the round's result on its lean: OD-066 (3N + 5 with every term attributed; the journal tail read is 10 documents and 6,010 bytes, 29 percent of the documents read on ten lines), OD-120 (Probe 7's states) and OD-154 (3 documents and 918 bytes for the grant read) for the convex lens, and OD-081 (Probe 7's restore answers) for the operator lens. OD-057 has left this list: its new lean closes on Probe 6's result and is checked. The session may not decide a `delegated` row (`decisions.py`: "session may decide only tactical rows"). +`delegated` rows that wait for the next sort of their lens, each with the round's result on its lean: OD-066 (3N + 5 with every term attributed; the journal tail read is 10 documents and 6,010 bytes, 29 percent of the documents read on ten lines), OD-120 (Probe 7's states) and OD-154 (3 documents and 918 bytes for the grant read) for the convex lens, OD-081 (Probe 7's restore answers) and OD-135 (its check) for the operator lens, and OD-095 (its check) for the domain lens. The session may not decide a `delegated` row (`decisions.py`: "session may decide only tactical rows"). -The 86 `delegated` rows carry the leans of the first panel run, five of them replaced in the tooling unit. A `gpt-6.1-sol` check read the first 23: twelve held on every checked claim and carry `checkedBy`; eight hold with a gap and three do not hold (OD-066, OD-103, OD-109), each waiting for its advisor's correction with the check's findings in the unit folder the project notes name. The tooling unit's recheck added four rows that carry `checkedBy` (OD-057, OD-071, OD-075, OD-080), so sixteen carry it; six of them are now decided, OD-010 and OD-014 at the tooling close and those four in 3b. The other leans are unchecked. A checked lean is applied to the Specs as a ruling by a later unit, one Spec family at a time, and the row becomes `decided`. The rest carry a one-line lean or wait for a probe, a measurement or a build (`--waiting`). +The 86 `delegated` rows carry the leans of the first panel run, five of them replaced in the tooling unit. A `gpt-6.1-sol` check read the first 23: twelve held on every checked claim and carry `checkedBy`; eight hold with a gap and three do not hold (OD-066, OD-103, OD-109), each waiting for its advisor's correction with the check's findings in the unit folder the project notes name. Later checks added `checkedBy` to OD-057, OD-071, OD-075 and OD-080 in the tooling unit and to OD-095, OD-135 and OD-149 in the close of round 3, so eighteen rows carry it, and eight of them are decided. The other leans are unchecked. A checked lean is applied to the Specs as a ruling by a later unit, one Spec family at a time, and the row becomes `decided`. The rest carry a one-line lean or wait for a probe, a measurement or a build (`--waiting`). ## Leads @@ -377,7 +366,7 @@ A session confirms or drops each. None is a finding yet. 4. The local backend refuses writes past 4 MiB a second. That bounds what slice S3 can measure on it, and a throughput number from it states the limit. 5. Convex ships `getConvexSize` and `getDocumentSize`. They answer the two byte-bound findings of S1. And `ctx.meta.getTransactionMetrics()` counts documents read and written across nested and component calls, in queries and mutations: S0 used it in two probes, and it is a cheap way for S1 to measure what a command reads. 6. Fifteen places where the design or a proposed fix repeats a shape the earlier platform regretted, each with the Spec line: progress kept on a row every command reads, work repeated per entry or per boundary, a stored maximum version taken as proof of coverage, a closed state with no route out, bookkeeping on the healthy path with no bounded lifetime, nested command calls where a helper would do, a recovery scan blocked by healthy work, registries with no reader. -7. Nine type names used in pinned signatures and declared by no Spec: `Journal`, `GetArgs`, `ListArgs`, `HistoryArgs`, `WriteBaselineArgs`, `WriteBaselineResult`, `DiagnosticSink`, `AuditRecordInput`, `BatchCursor`. Recipe 23 finds where each is used. +7. Four type names used in pinned signatures and declared by no Spec: `HistoryArgs`, `WriteBaselineArgs`, `WriteBaselineResult`, `BatchCursor`. Recipe 23 finds where each is used. 8. `dispatchId` on the obligation row is `v.id("_scheduled_functions")`. Nothing shows whether such an ID validates after a restore into another deployment. It belongs in Probe 7. 9. The mention audit (recipe 22) gives 78 unbacked pairs, 50 of them declared from neither side. Each slice checks the ones in its families. 10. Using the Protocol more, from a side thread's read on 2026-10-01. A local query for the build backlog at `defined` (recipe 1 reads stated `ready`, which no Spec here has before its slice passes; the same question at `defined` with a clear floor gave 80 Specs). A Pack per slice, so that "which Specs does S1 build" is a graph read. `check.py` counting open questions through recipe 20 in place of its own expression. The carrier grammar restated in `PLAN.md` 6.2, which can drift from the authoring skill's table. @@ -401,7 +390,7 @@ A session confirms or drops each. None is a finding yet. 23. Left by the platform review unit on purpose. Each is a hole in the handling of malformed input, a test gap, or a Spec line about what is not built. - The acceptance check: a run record whose backend entry is `{}` or names an unknown composition can still pass; three checks of a record's entries have no test of their own; an error that cannot be turned into text escapes the catch; `scripts/acceptance.mjs` imports the harness outside any `try`, so an import failure exits 1; an opening line with a full stop inside it is refused; deleting a test together with its anchor turns a passed scenario into a missing one and CI stays green. - The boundary: data with an own `code` and an inherited `kind` becomes a plain error. No Spec says which producer may send which cause variant, and nothing built checks it. "Validates as Convex validates a value" claims more than `validate` of `convex-helpers` with a plain-object check does. - - Not built, and the Specs pin them: the tombstone member of `ReceiptClass` and the tombstone flow; the baseline payload allowance (`append` holds a `baseline` event to 16,384 bytes like any other); `maxFoldEvents`; the receipts sweep; the largest-payload and fold-time measurements of the first experiment. + - Not built, and the Specs pin them: the tombstone member of `ReceiptClass` and the tombstone flow; the baseline payload allowance (`append` holds a `baseline` event to 16,384 bytes like any other); `maxFoldEvents`; the largest-payload and fold-time measurements of the first experiment. - Restore's check batch: a batch that folds several streams reads one stored baseline for each, and at four streams of 512 KiB that is above the 16 MiB ceiling. The Spec states the total for one stream and carries the open question; `r3-convex-restore-check-omits-baseline-reads` owns the batch shape. - Spec marks: numeric bullets that cite an E-number without the `[extension]` mark remain in the Specs the unit touched (`journal:94` and `:117`, `idempotency:101`, `queries:89-92`, `rebuild:124`, `restore:109-110`, `tenancy:78`, `persistence-adapter:106-107`), and four newly marked bullets have no matching open question in their own Spec. `check.py` tests neither. - Tests of S2 that sol's read showed would not catch a removal: the first-experiment order tests never offer an oversized row, the ten-line test counts writes and not their contents, the production-configuration lint assertion passes if the rule is removed from the configuration, and `codePathHolds` does not require one completion per call. In `tests/pure/evidence.test.ts` the test named for simultaneous writes writes one after the other. @@ -416,7 +405,7 @@ A session confirms or drops each. None is a finding yet. 27. From the roadmap unit. `ROADMAP.md`, "Findings beyond the roadmap", lists seven things the advisors found beside their task: the order summary's status validator admits a `cancelled` no event sets; `@convex-dev/migrations` is not pinned; OD-048's `reason` cites a line of `docs/modern-ts.md` as `package.json`; the ledger's `files` fields mix two path prefixes; `CONTEXT.md` names three fixture names where the fixture has five; no harness member exports or imports a snapshot. Each names the unit that takes it. Three of the four questions became rows in round 1, OD-151 to OD-153; the fourth is OD-046 and OD-064 in plain words. The memos' words that `CONTEXT.md` lacks join OD-009 and OD-022. -28. From round 1. `design/specs/application/rebuild.sdp.md` near line 45 promises a one-write rollback and near line 84 a reverify and switch: the fork of OD-068 shows both, and S4's paper rules it. The pipeline accepts a stock stream row at version zero with positive stock and no event, which no command makes and the version-zero comparison planted. The journal tail read and the grant read stay, by C3 and C4 of round 2, and removal needs the writer-invariant proof first. OD-066 waits for its lens with 3N + 5 attributed. Probe 7 closed three items of this lead: `writeTable` is exercised against component tables, a ZIP reader replaced `unzip`, and `BackendFacts.composition` is `null` after a temporary deploy. `snapshot` is on the avoid list of `CONTEXT.md` and the restore Spec and the harness use it. The f15 remainder names a work stage and a run date, which overlaps the temporal sweep of leads 16 and 22. `design/PLAN.md` near lines 297 and 299 still lists one example for Sc L2-2 and L2-4. The harness's child runner now has a both-streams output whose timeout reports through the bounded error and not the "did not finish" message. +28. From round 1. `design/specs/application/rebuild.sdp.md` near line 45 promises a one-write rollback and near line 84 a reverify and switch: the fork of OD-068 shows both, and S4's paper rules it. The pipeline accepts a stock stream row at version zero with positive stock and no event, which no command makes and the version-zero comparison planted. The journal tail read and the grant read stay, by C3 and C4 of round 2, and removal needs the writer-invariant proof first. OD-066 waits for its lens with 3N + 5 attributed. Probe 7 closed three items of this lead: `writeTable` is exercised against component tables, a ZIP reader replaced `unzip`, and `BackendFacts.composition` is `null` after a temporary deploy. The f15 remainder names a work stage and a run date, which overlaps the temporal sweep of leads 16 and 22. `design/PLAN.md` near lines 297 and 299 still lists one example for Sc L2-2 and L2-4. The harness's child runner now has a both-streams output whose timeout reports through the bounded error and not the "did not finish" message. 29. From round 2. Each is a test gap, a hole in a failure path, a Spec line about what is not built, or a number still to take. - `design/tools/test_decisions.py` has one test that fails at `main` too: `test_real_register_has_unique_rows_and_exact_recipe_20_coverage` ends in an `IndexError`. It is not one of the gates. @@ -424,9 +413,9 @@ A session confirms or drops each. None is a finding yet. - The Probe 6 context callback at `fixture/convex/migrations.ts:198-203` writes a `draft` summary beside the source's version whatever the source's status. Fixture probe code that nothing rests on; fixed only if that fixture is reused. - A partial tenant list passes `startGeneration`'s refusal: a list that holds a tenant whose first grant came after the table, and misses an older one, gets through. The rebuild Spec states it as an open question and tells the operator to run `fillTenants` first. `limitTenantFillBatch` is 8 because a grant row is bounded only by the 1 MiB document limit. - An event payload above the 16,384 bytes `append` holds it to reaches the caller as a bare technical failure: `append` throws a plain error inside the context, and nothing is stored. The example's stock item ID bound now refuses the case that reached it first; any command whose payload can outgrow the bound meets the same failure. - - From the gate: a duplicate is answered from its receipt in a closed scope and under the restore door, as the Spec now says; the plain error for an audited declaration with no subject has a built message and no pinned one; `baseline-operations.broken-audit-aborts` stays unbound until roadmap 4a builds an audited command. + - From the gate: a duplicate is answered from its receipt in a closed scope and under the restore door, as the Spec now says; the plain error for an audited declaration with no subject has a built message and no pinned one. - From `CancelOrder`: an invariant that exempts a cancelled order from its time and total checks survives every test; the bounds of the two context operations, Orders' 32 streams and Inventory's 100, have no direct `convex-test` case; `CancelOrder` declares no `bounds`, so a long order ID is refused by the adapter at 256 bytes after authorization and the receipt read. - - From the S4 paper: E-48 and E-49 are numbers of package D's range on a package C Spec, because C's range is full; the journal's baseline driver is not written as an operator entry, and the baseline writer changes sources without maintaining read models; the receipts sweep loops with `nextTenant`, which `receipt-table.sdp.md` does not say yet; a scheduled mutation that loses a race is assumed to rerun until it commits, and no pinned package shows a bound. + - From the S4 paper: E-48 and E-49 are numbers of package D's range on a package C Spec, because C's range is full; the journal's baseline driver is not written as an operator entry, and the baseline writer changes sources without maintaining read models; a scheduled mutation that loses a race is assumed to rerun until it commits, and no pinned package shows a bound. - From the probes, not run: Probe 6's timeout, retry exhaustion, a restore with a component worker pending and `parallelize`; Probe 7's hosted dashboard restore, the seven-day expiry, Workpool and Workflow (OD-115), file storage, a schema change between export and import, the Layer 2 restore drill, the argument limits of an action, and the limit of one million outstanding schedules. Leads 30 to 38 are lane data's of the tooling unit, as `leads-data.md` in its folder words them. @@ -449,12 +438,10 @@ Leads 30 to 38 are lane data's of the tooling unit, as `leads-data.md` in its fo 38. The timings the earlier platform took on 2026-09-26 (`convex` 1.46.0, a Docker backend) say nothing about the pinned release here, and F19 already records this release's system-operation refusal apart from the one-second execution limit. A reproduction under a recorded environment and load would settle any remaining question about time, and it would record times, not assert a call count or a start time as a limit. -39. From the hosted lane and its setup. Each item rests on source, docs or the setup run; no hosted test has run. - - Whether the table `plantedSchedules` of the hosted copy keeps its documents when the production composition, whose schema leaves it out, is deployed between two runs. No source in the hosted report shows it. One local run settles it: the proposed example `native-harness.undeclared-table-survives-another-composition-deploy`. If the rows are lost, the acceptance example deploys a copy of the production composition that declares the table. +39. From the hosted lane and its setup. Each item rests on source, docs, the setup run or the local backend; no hosted test has run. + - Whether the table `plantedSchedules` of the hosted copy keeps its documents when the production composition, whose schema leaves it out, is deployed between two runs: on the local backend it does, by `native-harness.undeclared-table-survives-another-composition-deploy` (native backend); the first hosted run shows it on the deployment. - `--expiration none` for a development deployment: settled. The setup lane's `deployment create` with CLI 1.46.0 accepted it on 2026-10-03. - - The deployment URL. The CLI printed `https://wandering-mule-403.eu-west-1.convex.cloud`, with the region, while the hosted patch's rule refuses a URL that is not `https://`, the key's deployment name and `.convex.cloud`. Either the rule takes the printed form, or a run shows the short form answers. The setup did not try the short form. - Serial runs are not enforced. A GitHub concurrency group does not serialize a developer's run against CI's, and one machine cannot see another's run. So the record names who ran it, `ranBy`, and a difference of two usage reports is the run's own only when no other run overlapped it. - - Redaction misses two paths until the build adds them: the HTTP and socket clients built with no logger (`harness/admin.ts`), and an aborted child call, which rejects with its abort reason unredacted (`harness/child.ts:83`). - That a development deploy key with the fourteen actions covers every CLI call the harness makes, export and import included, is from source only; the first hosted run settles it. 40. From the check of the 3b memo, beside what rulings 2 took. - `fixture/convex/migrations.ts:51` spreads the projection's result after a `!`. TypeScript erases the `!`, so a `null` fails at the write and not there. Line 39 is a `typeof` type query, so the rename there changes a type expression too. @@ -467,6 +454,18 @@ Leads 30 to 38 are lane data's of the tooling unit, as `leads-data.md` in its fo - What sol's last read left at `de285bb`: four mutations of the cursor codec survive (the `"undefined"` decoding, `jsonToConvex`, `convexToJson`, `sameValue`'s structural comparison), each unobservable while every composition's prefix is primitives; the codec's citation of `convex-helpers` ends at `stream.js:1447`, not `:1441`; the comment at `src/read-model/queries.ts:29` says "before a read" where the refusal follows the authorization and generation reads, and the `Value` type there does not admit the `undefined` sentinel; `read-models.sdp.md:78` says "either" over three fields. - Four mutations survive the review: M06, an abort that no longer reopens gates, unreachable while the paused path is unbuilt; M25, the fixture's `rebuild` module removed while `snapshot-replacement` stays green, because that test installs no read model; M31, the left-behind progress count forced to zero at `tests/native/place-order-measurement.ts:257`, since `rowsLeftBehind` is never checked against it; M32, the three new tables dropped from Probe 7's fresh-import comparison at `tests/native/scheduler-composition.ts:121`, which stores no rows in them. +42. From sitting two, left after lane 4f brought the refusal field, the tier names and the backup archive names to the Specs. + - The on-load migration is unbuilt either way: `src/context/adapter.ts` rejects a schema-version mismatch with a plain error, while the ruled answers are the state-only chain of step 3a and, for a history-reading step, the transient `writePaused` of OD-041. + - The words without a ruling are the Owner queue's language rows. +43. From the close of round 3. Each is a gap the build of its subject takes, a test gap, a number to take again, or a failure to read again. + - The history code gap: step 9 has no history or aggregate path, so the pipeline writes no history projection although the Specs pin one `applyHistoryProjection` call per command for each bound history read model. The history view's build (5d) takes it. F9's code side, `ReadModelBinding` with `AggregateProjection`, waits for OD-075. + - Probe 4's 1 s timeouts under load: its component and nested files time out at the one-second execution limit on the bytes-written step. They failed in the integrator's whole-tier run at `e97db90` and passed alone there, and failed in the run on `c1bdf80` and again alone at a load of 13 to 18. In CI at `d2a8d1b` the nested file timed out at 1 s in its Given step, which writes documents of 900 KiB, so the sensitivity is not only this machine's. Two lint tests in `tests/pure/compositions.test.ts` time out at their 5 s under the same load. + - Sc L2-5 in CI at `d2a8d1b`: `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` and the run threw `OptimisticConcurrencyControlFailure` at the interrupt step. 3b's F1 counted the interrupt's reruns as designed; whether a long batch chain can starve the interrupt, and what the operator then does, has no bound yet. The local whole-tier runs of this unit passed Sc L2-5. + - The hosted examples assume that a backup archive holds every mounted component's tables (F26, F41); the first hosted run shows it. + - What sol's read of the fold-in left. F38's 300 s deadline bounds when the usage polling admits a request and not when it completes, since the last usage request has its own 30 s timeout. F7 assigns the `awaitingBaseline` resolution to `finishRestoreRun` while the chain restarts after acceptance, and `fnFinishRestoreRun` handles no such resolution: paper for the restore's build (5b). F18's equality test with 1,000 versions would not catch a return to the linear search: a cost, not a correctness, check. + - After the follow-up `651fcc3`, an error whose `cause` is cyclic comes back as a redacted stub rather than a redacted copy; a visited set would keep its message. + - OD-093's provisional reading quotes the old E-9 text, and the register tool does not patch that field. + The reports behind leads 1 to 3 and 6 are outside the repository, because they quote a private repository. The project notes file named in `AGENTS.md` lists them, with the reports, briefs and rulings of S0. ## Inputs @@ -487,9 +486,3 @@ The doc is `docs/convex-transactional-domain-platform-decisions.md`, unchanged s - `.claude/agents/fable-xhigh.md` is the agent definition the round-3 reviewers ran as. - `.claude/agents/advisor-convex.md`, `advisor-domain.md`, `advisor-operator.md` and `advisor-product.md` are the definitions of the four advisors. Each holds three paths and nothing else. - `design/generated/` is ignored and regenerated by `npx sdp view design`. -42. From sitting two. Each is a gap between a ruled Spec and the code, taken by lane 4f, or a word still without a ruling. - - The refusal field: the Specs say `entry`, the code and its tests say `commandType` (`src/command/outcome-boundary.ts`, `src/command/pipeline.ts`, `src/command/authority.ts`, the compositions' queries and about ninety test files). `commandType` stays on receipts, declarations, the idempotency key, audit records, the envelope's cause and the dispatcher. - - The tier names: sixty example opening lines say `domain tier`, `build tier`, `native tier` or `end to end tier`, and `harness/acceptance.ts` with `tests/pure/acceptance.test.ts` parses those words; the ruled names are pure test, compiled, native backend, with composition and target apart. Eight non-example Specs and `design/README.md:133` use the old names too. The vitest project names (`pure`, `simulator`, `native`, `types`) are the harness's and stay. - - The harness's `snapshot` members (`harness/snapshot.ts`, `exportSnapshot`, `replaceSnapshot`, `deploySnapshotFixture`, the probe-7 and `snapshot-replacement` tests): the Specs say backup archive. The hosted unit rewrites those paths and takes the names with it; the restore Spec's own "snapshot" sentences are read as backup archive until that unit edits them. - - The on-load migration is unbuilt either way: `src/context/adapter.ts` rejects a schema-version mismatch with a plain error, which is the ruled answer for a history-reading step and not yet the state-only chain of step 3a. - - Words without a ruling, which keep the code's names: interrupt, park, stall, batch chain, fill, the aggregate's marker, history view and history projection, tombstone, drill, door, gate, audit record, diagnostic, adopter, package, activation record. From 4ff759d186da40d35776b9e4526e33f5fa9e25e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 11:22:07 +0200 Subject: [PATCH 149/155] build: Probe 4 reaches the bytes-written limit inside its time budget The blobs payload is an ASCII string, not bytes: convex/values converts bytes to base64 in JavaScript, which took the combined write 0.44 s of its 1 s budget unloaded. The string takes 0.065 s at the same document size and still fails on the bytes-written limit. The test measures the combined write's execution time from the function log. Co-Authored-By: Claude Opus 5.5 --- ...mits.probe-4-component-call-shares-limits.sdp.md | 1 + ...-limits.probe-4-nested-call-shares-limits.sdp.md | 1 + fixture/convex/annex/limits.ts | 4 ++-- fixture/convex/annex/schema.ts | 2 +- fixture/convex/limits.ts | 6 +++--- fixture/convex/schema.ts | 4 +++- tests/native/shared-limits.ts | 13 ++++++++++++- 7 files changed, 23 insertions(+), 8 deletions(-) diff --git a/design/specs/facts/f13-transactions-have-limits.probe-4-component-call-shares-limits.sdp.md b/design/specs/facts/f13-transactions-have-limits.probe-4-component-call-shares-limits.sdp.md index 49e9376..d665e94 100644 --- a/design/specs/facts/f13-transactions-have-limits.probe-4-component-call-shares-limits.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.probe-4-component-call-shares-limits.sdp.md @@ -29,4 +29,5 @@ And the same split of bytes written by a parent mutation and a component mutatio - The test also reads `ctx.meta.getTransactionMetrics()` in the parent before and after a component read that stays under the limit and asserts that `bytesRead.used` and `documentsRead.used` grew by what the component read. - The test also shows that each half of the write commits alone, so the failure is the sum. - The local backend refuses writes past a rate, with `TooManyWrites` and the text "limited to 4 MiB bytes written per 1 second", so the test paces the transactions that seed and write its documents. +- Each document holds an ASCII string, whose length is its size in bytes. Bytes would spend most of a mutation's 1 second of execution under load, because `convex/values` converts them to base64 in JavaScript, and the run would show the time limit instead of the bytes-written limit. - On the first run of this example, on 2026-10-01 on release `precompiled-2026-09-28-5c7cb5b`, the bound values held. diff --git a/design/specs/facts/f13-transactions-have-limits.probe-4-nested-call-shares-limits.sdp.md b/design/specs/facts/f13-transactions-have-limits.probe-4-nested-call-shares-limits.sdp.md index 383dbe4..92c7f1d 100644 --- a/design/specs/facts/f13-transactions-have-limits.probe-4-nested-call-shares-limits.sdp.md +++ b/design/specs/facts/f13-transactions-have-limits.probe-4-nested-call-shares-limits.sdp.md @@ -29,4 +29,5 @@ And the same split of bytes written by a parent mutation and a nested mutation { - The test also reads `ctx.meta.getTransactionMetrics()` in the parent before and after a nested read that stays under the limit and asserts that `bytesRead.used` and `documentsRead.used` grew by what the child read. - The test also shows that each half of the write commits alone, so the failure is the sum. - The local backend refuses writes past a rate, with `TooManyWrites` and the text "limited to 4 MiB bytes written per 1 second", so the test paces the transactions that seed and write its documents. +- Each document holds an ASCII string, whose length is its size in bytes. Bytes would spend most of a mutation's 1 second of execution under load, because `convex/values` converts them to base64 in JavaScript, and the run would show the time limit instead of the bytes-written limit. - On the first run of this example, on 2026-10-01 on release `precompiled-2026-09-28-5c7cb5b`, the bound values held. diff --git a/fixture/convex/annex/limits.ts b/fixture/convex/annex/limits.ts index 884b87e..ce807cc 100644 --- a/fixture/convex/annex/limits.ts +++ b/fixture/convex/annex/limits.ts @@ -8,7 +8,7 @@ export const insertBlobs = mutation({ for (let i = 0; i < count; i++) { await ctx.db.insert("blobs", { group, - bytes: new ArrayBuffer(size), + payload: "x".repeat(size), }); } return null; @@ -34,7 +34,7 @@ export const readBlobs = query({ const after = await ctx.meta.getTransactionMetrics(); return { documents: documents.length, - payloadBytes: documents.reduce((sum, d) => sum + d.bytes.byteLength, 0), + payloadBytes: documents.reduce((sum, d) => sum + d.payload.length, 0), before, after, }; diff --git a/fixture/convex/annex/schema.ts b/fixture/convex/annex/schema.ts index ecb434b..32fc3f9 100644 --- a/fixture/convex/annex/schema.ts +++ b/fixture/convex/annex/schema.ts @@ -8,7 +8,7 @@ export default defineSchema({ notes: defineTable({ source: v.string() }), throwerWrites: defineTable({ by: v.string() }).index("by_by", ["by"]), samples: defineTable({ value: v.number() }), - blobs: defineTable({ group: v.string(), bytes: v.bytes() }).index( + blobs: defineTable({ group: v.string(), payload: v.string() }).index( "by_group", ["group"], ), diff --git a/fixture/convex/limits.ts b/fixture/convex/limits.ts index 6d70c52..5b2c965 100644 --- a/fixture/convex/limits.ts +++ b/fixture/convex/limits.ts @@ -23,7 +23,7 @@ export const insertBlobs = mutation({ for (let i = 0; i < count; i++) { await ctx.db.insert("blobs", { group, - bytes: new ArrayBuffer(size), + payload: "x".repeat(size), }); } return null; @@ -47,7 +47,7 @@ export const readBlobs = internalQuery({ const after = await ctx.meta.getTransactionMetrics(); return { documents: documents.length, - payloadBytes: documents.reduce((sum, d) => sum + d.bytes.byteLength, 0), + payloadBytes: documents.reduce((sum, d) => sum + d.payload.length, 0), before, after, }; @@ -96,7 +96,7 @@ export const writeThenCall = mutation({ for (let i = 0; i < a.parentCount; i++) { await ctx.db.insert("blobs", { group: a.group, - bytes: new ArrayBuffer(a.size), + payload: "x".repeat(a.size), }); } await ctx.runMutation( diff --git a/fixture/convex/schema.ts b/fixture/convex/schema.ts index 8743e51..14c45b8 100644 --- a/fixture/convex/schema.ts +++ b/fixture/convex/schema.ts @@ -33,7 +33,9 @@ export default defineSchema({ "by_position", ["position"], ), - blobs: defineTable({ group: v.string(), bytes: v.bytes() }).index( + // An ASCII payload, so its length is its size in bytes. Writing and reading it takes a small part + // of the execution time that bytes take, which `convex/values` converts to base64 in JavaScript. + blobs: defineTable({ group: v.string(), payload: v.string() }).index( "by_group", ["group"], ), diff --git a/tests/native/shared-limits.ts b/tests/native/shared-limits.ts index 4935731..a4c5d50 100644 --- a/tests/native/shared-limits.ts +++ b/tests/native/shared-limits.ts @@ -174,11 +174,13 @@ async function writtenRows(world: LimitsWorld, group: string) { return [...own, ...child].filter((row) => row.group === group).length; } export async function writeLimits(world: LimitsWorld, expected: string) { - const client = ordinaryClient(required(world.backend, "the backend").url); + const backend = required(world.backend, "the backend"); + const client = ordinaryClient(backend.url); const through = required(world.through, "the boundary"); const parentCount = required(world.parentCount, "the parent count"); const childCount = required(world.childCount, "the child count"); const size = required(world.size, "the size"); + const mark = await backend.admin.logMark(); const together = await outcome( client.mutation(api.limits.writeThenCall, { through, @@ -192,6 +194,15 @@ export async function writeLimits(world: LimitsWorld, expected: string) { "written", ); measure("writesTogether", together); + const records = await backend.admin.completionsSince(mark, (entries) => + entries.some((entry) => entry.identifier === "limits:writeThenCall"), + ); + // The local backend logs one completion for the call; the nested call has none of its own. + const execution = required( + records.find((entry) => entry.identifier === "limits:writeThenCall"), + "the combined write's completion", + ); + measure("combinedWriteSeconds", execution.executionTime); // Pace even a failed attempt before testing the two committing halves. await paceAfterWrite((parentCount + childCount) * (size + 256)); const failedRows = await writtenRows(world, "failed-write"); From 2a44655ab10b0e4c20d4a099b0ebdb58d58df380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 11:30:11 +0200 Subject: [PATCH 150/155] build: the interrupt touches the generation row only interruptGeneration reads and writes the generation row alone: it refuses verified, active and purged, bumps the fence and records who and when. The batch one fence below, the pending batch of the chain the interrupt replaced, records the interruption on the progress row with the operator from changedBy and schedules nothing, so an interrupt conflicts with a batch only through the generation row. CI at d2a8d1b showed the interrupt losing every rerun to back-to-back checkpoints. Co-Authored-By: Claude Opus 5.5 --- design/STATE.md | 2 +- .../application/generation-registry.sdp.md | 4 +- ...ild.online-rebuild-interrupt-resume.sdp.md | 6 +- design/specs/application/rebuild.sdp.md | 10 +- src/read-model/rebuild.ts | 30 ++-- ...ld.online-rebuild-interrupt-resume.test.ts | 36 +++-- tests/simulator/rebuild.test.ts | 135 +++++++++++++++--- 7 files changed, 168 insertions(+), 55 deletions(-) diff --git a/design/STATE.md b/design/STATE.md index 69309f7..a239f71 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -65,7 +65,7 @@ Round 3, unit 3b, `application-platform-round-3`, on 2026-10-03 from about 03:32 - The integration. An Opus integrator merged fix, proof and native with no conflict, codegen unchanged after each, in two stages while the native lane still ran. At the merge 325 of 325 proof tests passed after 19 wrong type tests were corrected (18 a misplaced `@ts-expect-error`, one the `| undefined` of `exactOptionalPropertyTypes`); the fix was wrong nowhere. It corrected three setups no lane rewrote, the L2-5 example's opening line, and seven native tests that counted the install's two batch rows. At `a013ba4`: `npm test` 943 passed; the native tier 153 tests, 128 passed, 20 failed, 5 skipped; acceptance 30 passed, 2 failed (Sc L2-5, Sc L2-9), 4 missing. The 20 were four questions for the main thread. - The cutover design. The L2-5 run's subscription threw `InvalidCursor` at the switch, because the built-in `paginate` refuses a cursor from another query and the active generation is in the range. One Fable designer in 7 minutes, with a spike at `convex-test`, showed the raw cursor of `paginator` failing three ways: a page-two cursor answered page one again, a pinned page threw `ConcatStreams in wrong order`, and after a switch back a cursor answered the retired generation's rows. The ruling: a parent list pages with `paginator` as a context list does, `by_status` ends with `key`, and `pageInGeneration` moves each cursor of the client's pair into the active generation. - The fold-in, one Opus agent in three passes, from rulings F1 to F8 and G1 to G5, then H1: - - F1: the interrupt and a running batch both patch the progress row and one reruns, as designed; the L2-5 sentence now says no live command or parent query names `generationProgress` in its `occInfo`, and the entries' and batches' reruns there are counted. + - F1: the interrupt and a running batch both patched the progress row and one reran, as designed then; after CI at `d2a8d1b` showed the interrupt losing every rerun, the interrupt touches the generation row only, so it conflicts with a batch only through that row and a batch that conflicts reruns once and stops on the new fence, and the batch the interrupt replaced records the interruption; the L2-5 sentence says no live command, parent query or interrupt names `generationProgress` in its `occInfo`, and the batches' and other entries' reruns there are counted. - F2: the cutover design applied in both compositions' lists and `src/read-model/queries.ts`, with the spike's four cases as `convex-test` cases. - F3: the slimmer row's observations rerun alone: 95 bytes fewer read per generation row read (190 with a building generation), 9 bytes more written per summary row because `by_status` ends with `key`. - F4: the left-behind schema check counts `generationProgress`, `tenantFill` and `projectionMarkers`, and Sc L2-9 passes. diff --git a/design/specs/application/generation-registry.sdp.md b/design/specs/application/generation-registry.sdp.md index bc89f3f..8c968ea 100644 --- a/design/specs/application/generation-registry.sdp.md +++ b/design/specs/application/generation-registry.sdp.md @@ -19,7 +19,7 @@ relations: Story · Detail: full · Traces: D8, D9, D11, D19, F1, F12, F13, Law 6, Law 11, E-8, E-9, E-40, E-42, E-48, E-49. -The registry is two parent tables. The `generations` table records every generation of every read model with its state, its projection version and its fence, and it is the row a command and a query read. The `generationProgress` table holds, per generation, the cursor, the batch size and the counts of the batches that build, verify and purge it, and only a batch and an operator entry read it. A generation is a build of the projection's code, which is the same for every tenant, so one generation row spans every tenant of the deployment and its batches iterate the tenants; there is no generation of one tenant and no rebuild for one tenant. A use case reads the registry once per read model it maintains to learn which generations to write; a query reads it once to learn which generation to read; the rebuild owns the rows. The registry changes only at a transition of a generation, so it is not an application-wide counter in the write path. +The registry is two parent tables. The `generations` table records every generation of every read model with its state, its projection version and its fence, and it is the row a command and a query read. The `generationProgress` table holds, per generation, the cursor, the batch size and the counts of the batches that build, verify and purge it, and only a batch and an operator entry other than `interruptGeneration` read it; the interrupt reads and writes the generation row alone, so it conflicts with a batch only through that row. A generation is a build of the projection's code, which is the same for every tenant, so one generation row spans every tenant of the deployment and its batches iterate the tenants; there is no generation of one tenant and no rebuild for one tenant. A use case reads the registry once per read model it maintains to learn which generations to write; a query reads it once to learn which generation to read; the rebuild owns the rows. The registry changes only at a transition of a generation, so it is not an application-wide counter in the write path. ## Intent @@ -74,7 +74,7 @@ Tables live in the parent's `schema.ts`, spread from the library's `readModelTab - validatorProgressPass: `const progressPassValidator = v.union(v.literal("backfill"), v.literal("verify"), v.literal("purge"), v.literal("idle"))` naming the pass a batch chain is driving, `idle` when none is (E-40) - tableGenerationProgress: `generationProgress: defineTable({ generationId: v.id("generations"), pass: progressPassValidator, batchSize: v.number(), cursor: v.union(batchCursorValidator, v.null()), batchesDone: v.number(), rowsWritten: v.number(), rowsSkipped: v.number(), misses: v.number(), rowsPurged: v.number(), lastError: v.optional(v.string()), updatedAt: v.number() })` one row per generation, inserted by `startGeneration` in the mutation that inserts the generation row; `cursor` is the position in the pass that `pass` names and `null` before that pass has read its first tenant; `batchesDone` counts every batch of the generation that reached its checkpoint, the batch that ends a pass included and a parked batch not, `rowsWritten` the rows a backfill batch inserted, updated or deleted, `rowsSkipped` the items a backfill or verify batch left as they were, `misses` the rows the current verify pass inserted, updated or deleted, and `rowsPurged` the rows a purge deleted; every write of the row sets `updatedAt` to the writing mutation's `Date.now()` (E-40, D9, F13) - indexProgressByGeneration: `.index("by_generation", ["generationId"])` (E-40) -- indexProgressByGenerationUse: the one read a batch and an operator entry make to find a generation's progress row; no command and no query over a read model reads the table (E-40, F1) +- indexProgressByGenerationUse: the one read a batch and an operator entry other than `interruptGeneration` make to find a generation's progress row; no command and no query over a read model reads the table (E-40, F1) - tableProjectionMarkers: `projectionMarkers: defineTable({ tenantId: v.string(), readModel: v.string(), generation: v.number(), entityKey: v.string(), aggregateKey: v.string(), contribution: v.number(), sourceVersions: v.array(streamVersionValidator) })` (E-8, D9, Law 11) - indexMarkersByEntity: `.index("by_entity", ["tenantId", "readModel", "generation", "entityKey"])` (E-8) - indexMarkersByEntityUse: point read by `applyAggregate` before every aggregate write, with the `entityKey` it derives from the entity's source stream, `JSON.stringify([contextId, streamType, streamId])` of the input's stream version, as `spec:application.projection-contract` states (D9, E-8, E-9) diff --git a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md index c72d1e8..aa0e108 100644 --- a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md +++ b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md @@ -30,9 +30,9 @@ And writes {writes: "never stopped"} ## Verification — executable - Runs in the native backend tier; every test owns its disposable backend. -- The test, in a file whose own timeout is 300 s, installs the order summary's generation 1 through its first rebuild, seeds 350 orders across two tenants, 250 in the first in tenant-ID order and 100 in the second, starts generation 2 with `projectionVersion` 1 stated, a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that `batchesDone` then stops advancing below 350 and that the cursor names a tenant; while a chain is expected to run, a progress row whose `batchesDone` has not advanced for 30 s fails the test with the generation row and the progress row in the message; once generation 2 is `verified` it asserts that the backfill and verify completions in the function log that wrote number the progress row's `batchesDone`, and that each pass took at least the 350 seeded orders and at most the orders at `verified` plus one per tenant. +- The test, in a file whose own timeout is 300 s, installs the order summary's generation 1 through its first rebuild, seeds 350 orders across two tenants, 250 in the first in tenant-ID order and 100 in the second, starts generation 2 with `projectionVersion` 1 stated, a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that the batch the interrupt replaced then sets the progress row's `lastError` to `"interrupted by "` and the interrupt's operator, that `batchesDone` stops advancing below 350 and that the cursor names a tenant; while a chain is expected to run, a progress row whose `batchesDone` has not advanced for 30 s fails the test with the generation row and the progress row in the message; once generation 2 is `verified` it asserts that the backfill and verify completions in the function log that wrote number the progress row's `batchesDone` plus the one backfill batch that recorded the interruption, and that each pass, that batch aside, took at least the 350 seeded orders and at most the orders at `verified` plus one per tenant. - While the chain is interrupted the test sends `CancelOrder` to one of the second tenant's 100 seeded orders and to an order of the first tenant whose row generation 2 already holds, read from the table before the command; it asserts at once that the first tenant's order has its row in generation 2 `cancelled`, and, once generation 2 is `verified`, that the second tenant's order has its row in generation 2 `cancelled`, whichever pass wrote it. - After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that, once the generation is `verified`, every order the Orders context's `list` enumerates in either tenant, read with admin access and `includeDeleted`, has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, that `backfillBatch` run with admin access under the fence the interrupt replaced leaves the progress row unchanged, that the function log shows no action and no completion during the batches other than a batch, an operator entry or a live command, and that the journals, read by operation, hold the event and receipt counts the live commands produced. -- The test asserts that no command of the run ends in a failure, that no completion record of a `PlaceOrder`, a `CancelOrder` or a parent query names `generationProgress` in its `occInfo`, and that the completion records whose `occInfo` names `generations` number at most the operator entries and pass ends of the run times the callers the test keeps in flight plus one for the chain, and records those reruns with their `occInfo` as a measurement, beside them the count of the reruns of operator entries and batches whose `occInfo` names `generationProgress`, each with its `occInfo`; it reads generation 2's row at the interrupt, after the resume and at `verified`, and asserts that the interrupt and the resume changed only `fence`, `changedAt` and `changedBy` and that from the resume to `verified` only `state` changed. +- The test asserts that no command of the run ends in a failure, that no completion record of a `PlaceOrder`, a `CancelOrder` or a parent query names `generationProgress` in its `occInfo`, that no completion record of `interruptGeneration` names `generationProgress` either, since the interrupt conflicts with a batch only through the generation row and a batch that conflicts reruns once and stops on the new fence, and that the completion records whose `occInfo` names `generations` number at most the operator entries and pass ends of the run times the callers the test keeps in flight plus one for the chain, and records those reruns with their `occInfo` as a measurement, beside them the reruns of batches and of the other operator entries whose `occInfo` names `generationProgress`, each with its `occInfo`; it reads generation 2's row at the interrupt, after the resume and at `verified`, and asserts that the interrupt and the resume changed only `fence`, `changedAt` and `changedBy` and that from the resume to `verified` only `state` changed. - The test switches, asserts from the function log that the list query ran again for each tenant's subscription after the switch and that each subscription's value holds the same rows, and that generation 1 is `retired`, then runs one more `PlaceOrder` and cancels one earlier order so that its summary changes, rolls back by naming generation 1, asserts that the reopened generation reaches `verified` after one verify pass with a row for the new order and the cancelled order's row `cancelled` before it becomes active, switches again, and asserts that generation 1 is active, generation 2 is `retired` and the list query ran again for each subscription. -- A `convex-test` test of the same workflow runs on the fixture composition with `documentTitle` and calls each batch directly, so it interrupts after exactly 3 batches; in one run generation 1 is installed at version 1 stated and generation 2 built at version 2 under live `CreateTwiceSummarizedDocument` writes, and it asserts that each generation's rows carry their own generation's `projectionVersion` and that a document whose `title` is the depot's `deletedTitle` has a row at version 1 and none at version 2; in a second run both generations are at version 2 and, after the switch, it deletes a document by `AmendDocument` with the depot's `deletedTitle`, rolls back by naming generation 1 and asserts that the reopened generation holds no row for it after its verify pass; and it asserts every refusal of the operator entries by its message. +- A `convex-test` test of the same workflow runs on the fixture composition with `documentTitle` and calls each batch directly, so it interrupts after exactly 3 batches, and asserts that an interrupt issued while a batch is scheduled leaves the progress row untouched and that the scheduled batch records the interruption and schedules nothing; in one run generation 1 is installed at version 1 stated and generation 2 built at version 2 under live `CreateTwiceSummarizedDocument` writes, and it asserts that each generation's rows carry their own generation's `projectionVersion` and that a document whose `title` is the depot's `deletedTitle` has a row at version 1 and none at version 2; in a second run both generations are at version 2 and, after the switch, it deletes a document by `AmendDocument` with the depot's `deletedTitle`, rolls back by naming generation 1 and asserts that the reopened generation holds no row for it after its verify pass; and it asserts every refusal of the operator entries by its message. diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index ae1b70e..7ec52a5 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -88,11 +88,11 @@ The registry, the progress row, the per-entity marker and the gate document are - rule: [extension] Every batch, whether its source is a context's enumeration or a parent table, is driven by a self-scheduled internal mutation under one batch contract, and the rebuild uses `@convex-dev/migrations` for nothing (E-40, F17, F8) - rule: [extension] A projection version change is a new generation, never an in-place rewrite of the active one (E-8, D8) - rule: [extension] The batch bound is a field of the progress row, the read model's by `batchSizeFor`, a whole number of at least 1 and at most `rebuildBatchCeiling(readModel)`; the context's `list` page bounds the bytes a batch reads, so a batch may take fewer entities than its size, and does not change the size; `startGeneration` sets it and nothing changes it after, so every pass and every resumed pass runs at the size on the row, and a purge batch deletes at most that many rows, capped by `limitPurgeBatch(readModel)` (E-40, E-24, E-9, F1, F13) -- rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them returns without writing (E-40, E-8, F8) +- rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them checkpoints nothing and schedules nothing; the pending batch of a chain `interruptGeneration` replaced writes only the interruption, on the progress row (E-40, E-8, F8) - An operator calls `startGeneration` for a read model; it inserts the generation row in state `building` with the next generation number, the projection version and fence 1, inserts the progress row in the pass `backfill` with the batch size and a `null` cursor, and schedules the first backfill batch, all in one mutation; a read model with no generation gets generation 1 this way (D9, F8, E-8) - [extension] On a deployment whose grants predate the tenant list an operator calls `fillTenants` before the first `startGeneration`; it inserts the `tenantFill` row, or reopens it, in the pass `fill` with a new fence and schedules the first fill batch; each fill batch reads the gate, takes up to `limitTenantFillBatch` further tenants from the grants after the row's cursor, inserts the tenant row of each that has none, saves the last tenant ID as the cursor and schedules the next fill batch, and the batch that finds no further tenant sets the pass to `idle` and schedules nothing (E-49, E-48, E-40, F8) - From that commit every use case that maintains the read model writes the active generation, when there is one, and the one being built through `applyProjection`, in live-created mode for a stream it created and live-updated mode otherwise (D9, E-9) -- Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building`, the pass is not `backfill` or the fence differs, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) +- Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building` or the pass is not `backfill`, returns when the fence differs, after recording an interruption as step 1 states, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) - A batch that throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, and a developer error or a timeout ends the chain; the operator's `interruptGeneration` ends a chain on purpose, and `resumeGeneration` schedules a batch from the saved cursor with a new fence, at the batch size on the progress row; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) - Each verify batch enumerates the sources again from the progress row's cursor, which the end of the backfill reset to `null`, the first tenant, and calls `applyProjection` in backfill mode for every DTO, so a missing row is inserted, an older one updated, and the row of a DTO that projects to `null` deleted; each such repair is counted as a miss; when the last tenant's last page is done the generation moves to `verified` and no batch is scheduled (D9, E-8, Sc L2-5) - `switchGeneration` sets the verified generation `active` and, when the read model has an active generation, sets that one `retired` with `retiredAt` and `retireAfter`, in one mutation; every query reads the active generation from the registry, so every subscription on the read model re-runs (D9, F7, F1) @@ -122,7 +122,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - fnVerifyBatch: `export const verifyBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9) - fnPurgeBatch: `export const purgeBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D19, E-40) - fnEnumerateCall: `ctx.runQuery(target.list, { tenantId, includeDeleted: true, paginationOpts: { numItems: batchSize, cursor: pageCursor } })` returning the `paginator` page `{ page, isDone, continueCursor }` of DTOs with their stream versions, one call per batch; the query caps `numItems` at its byte-derived bound, so a batch never reads more than 8 MiB of stream rows however `batchSize` was set (D2, D10, F15, S4, E-24) -- step1: `backfillBatch` loads the generation row by ID and its progress row by `by_generation`, and returns without writing when the row is absent, `state` is not `building`, `pass` is not `backfill` or `fence` differs from its argument (E-40, D9) +- step1: `backfillBatch` loads the generation row by ID and its progress row by `by_generation`, and returns without writing when the row is absent, `state` is not `building` or `pass` is not `backfill`; when `fence` differs from its argument it checkpoints nothing, schedules nothing and returns, and when its argument is one below `fence`, which after `interruptGeneration` makes it the pending batch of the chain the interrupt replaced, it first patches the progress row's `lastError` to `"interrupted by " + changedBy`, the generation row's, and `updatedAt`, unless `lastError` already says so; an interrupt therefore conflicts with a batch only through the generation row, and a batch that conflicts reruns once and stops on the new fence (E-40, E-48, D9, Sc L2-5) - step2: it takes its tenant: the cursor's `tenantId`, or `nextTenant(ctx, null)` when the cursor is `null`; when the tenant list is empty the batch has no tenant, reads the gate as step 3 states, and goes to step 6 with the pass done (E-49, Law 11) - step3: it calls the write pause's `gateAllows(ctx, ["all", "tenant:" + tenantId], generationId)`, with `["all"]` alone when it has no tenant, and when the answer is not allowed it parks: it patches the progress row's `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt`, schedules nothing and returns, so the generation waits for `resumeGeneration` after the gate reopens; no batch calls `assertWritable`, because a batch parks its generation rather than failing its scheduled mutation with a thrown refusal (E-8, E-42, D9) - step4: it calls `list` once as `fnEnumerateCall` states, with the cursor's `pageCursor` (D10, E-40) @@ -131,7 +131,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `pass` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) - purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(Math.min(batchSize, limitPurgeBatch(readModel)))` with `batchSize` the progress row's and `readModel` the generation's read model, whose cap `limitPurgeBatch` declares, deletes each row and adds their number to `rowsPurged`; when it deleted that many rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`, a purge cursor being `{ tenantId, pageCursor: null }`; a pass that is not done checkpoints, patching the progress row with the cursor, `rowsPurged`, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) - purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) -- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row's `lastError` to `"interrupted by " + operator` and `updatedAt`, and scheduling nothing, so the chain's next batch returns without writing and the generation stays at its checkpoint (E-40, E-48, D9, Sc L2-5) +- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, reading the generation row and nothing else, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `state` is `verified`, `active` or `purged`, in which no pass runs; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, and scheduling nothing; a `retired` or `aborted` generation runs a pass only during a purge, which its row does not tell, so the interrupt accepts either and the new fence changes nothing when no batch runs; it neither reads nor patches the progress row, so no checkpoint conflicts with it: the chain's pending batch records the interruption as step 1 states and the generation stays at its checkpoint, and a parked or stranded chain, which has no pending batch, keeps its progress row as it was (E-40, E-48, D9, Sc L2-5) - fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to no `lastError` and `updatedAt` and leaving its `batchSize`, cursor and counts as they are, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the backup archive stranded (D9, F8, E-40, E-42) - fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain`; it takes no batch size, so a resumed pass runs at the progress row's `batchSize`, a purge capped by `limitPurgeBatch`; for a generation in any state whose progress row names a pass, so it continues a backfill or verify pass of a generation in `building` or `verifying` and a started purge of a `retired` or `aborted` one, which is how an operator continues a purge a restore stranded (D9, Sc L2-5, E-40, E-48) - fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) @@ -149,7 +149,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch; a batch reads the generation row and never writes it outside the end of a pass, and it writes the progress row, which no command reads, so a batch's checkpoint re-runs no command (F1, D9, Probe 6, E-40) - limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch of a per-entity read model and events per batch of a history view, a history projection that declares no `rowBudgetBytes` counting 16 KiB, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) - limitPurgeBatch: `limitPurgeBatch(readModel: ReadModel | HistoryProjection): number`, `min(500, floor(4194304 / readModel.rowBudgetBytes))` rows, a history projection that declares no `rowBudgetBytes` counting 16 KiB, the cap on the rows a purge batch deletes, which deletes the progress row's `batchSize` when that is smaller; 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) -- limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch returns without writing (E-40, F13) +- limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch checkpoints nothing and schedules nothing, and writes at most the interruption step 1 records (E-40, F13) - limitBuildingGenerations: at most one generation in `building`, `verifying` or `verified` per read model at a time, so a command writes at most two generations, the active one and that one (E-8, E-9) - limitRollbackPeriod: `[extension]` 7 days by default, stored as `retireAfter` on the retired row; the retired generation is kept and not written, and a rollback at or after `retireAfter` is refused (E-8, D9) - limitGenerationsListed: 20 generations per `getGenerations` answer, the newest first, a library constant (E-8, F13) diff --git a/src/read-model/rebuild.ts b/src/read-model/rebuild.ts index 60d64f6..c89bfa0 100644 --- a/src/read-model/rebuild.ts +++ b/src/read-model/rebuild.ts @@ -229,6 +229,8 @@ export async function startGeneration( }); return generationId; } +// Reads and writes the generation row alone, so a batch's checkpoint never conflicts with it; the batch +// the new fence replaces records the interruption on the progress row. export async function interruptGeneration( parentCtx: GenericMutationCtx, _config: RebuildConfig, @@ -236,19 +238,18 @@ export async function interruptGeneration( ): Promise { const operator = assertOperator(args.operator); const ctx = mutationContext(parentCtx); - const { generation, progress } = await load(ctx, args.generationId); - if (progress.pass === "idle") + const generation = await loadGeneration(ctx, args.generationId); + if ( + generation.state === "verified" || + generation.state === "active" || + generation.state === "purged" + ) throw new Error(`${description(generation)} has no batch to interrupt`); - const now = Date.now(); await ctx.db.patch(generation._id, { fence: generation.fence + 1, - changedAt: now, + changedAt: Date.now(), changedBy: operator, }); - await ctx.db.patch(progress._id, { - lastError: `interrupted by ${operator}`, - updatedAt: now, - }); return null; } // Bumps the fence and schedules the batch of the pass the progress row names, in the caller's mutation. @@ -478,12 +479,15 @@ async function generationBatch( : pass === "verify" ? generation.state === "verifying" : generation.state === "retired" || generation.state === "aborted"; - if ( - !stateMatches || - progress.pass !== pass || - generation.fence !== args.fence - ) + if (!stateMatches || progress.pass !== pass) return null; + if (generation.fence !== args.fence) { + // The batch one fence below is the pending batch of the chain the interrupt replaced: it records + // the interruption once; a batch of an older chain writes nothing. + const lastError = `interrupted by ${generation.changedBy}`; + if (args.fence === generation.fence - 1 && progress.lastError !== lastError) + await ctx.db.patch(progress._id, { lastError, updatedAt: Date.now() }); return null; + } const tenantId = progress.cursor?.tenantId ?? (await nextTenant(ctx, null)); const gate = await gateAllows( ctx, diff --git a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts index f89c50c..a9a7dae 100644 --- a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts +++ b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts @@ -348,7 +348,8 @@ async function assertLog(w: World) { [], ); // rebuild.online-rebuild-interrupt-resume.sdp.md:33: the batches that wrote, by pass, are the progress - // row's batchesDone; a batch under a replaced fence completes without writing and is not counted. + // row's batchesDone and the one backfill batch that recorded the interruption; any other batch under a + // replaced fence completes without writing. const wrote = (identifier: string) => during.filter( (record) => @@ -357,7 +358,7 @@ async function assertLog(w: World) { record.error === null && (record.usageStats["databaseWriteDocuments"] ?? 0) > 0, ).length; - const backfills = wrote("rebuild:backfillBatch"); + const backfills = wrote("rebuild:backfillBatch") - 1; const verifies = wrote("rebuild:verifyBatch"); expect(backfills + verifies).toBe( required(w.verified, "verified generation").progress.batchesDone, @@ -390,6 +391,15 @@ async function assertLog(w: World) { (live.has(record.identifier) || record.udfType === "Query"), ), ).toEqual([]); + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: the interrupt conflicts with a batch only through the + // generation row, so no completion of it names the progress row. + expect( + log.filter( + (record) => + record.identifier === "rebuild:interruptGeneration" && + record.occInfo?.tableName === "generationProgress", + ), + ).toEqual([]); const commands = new Set([ getFunctionName(api.ordering.placeOrder), getFunctionName(api.ordering.cancelOrder), @@ -537,7 +547,7 @@ bindExample( registryReruns: records(w).filter( (record) => record.occInfo?.tableName === "generations", ), - // rebuild.online-rebuild-interrupt-resume.sdp.md:36: operator entries and batches that reran on the progress row. + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: batches and the other operator entries that reran on the progress row. progressReruns: records(w).filter( (record) => record.occInfo?.tableName === "generationProgress" && @@ -647,7 +657,16 @@ bindExample( { generationId: id2(w), operator: "interrupt operator" }, ); await stopLive(w); - w.interrupted = await entry(w); + // rebuild.online-rebuild-interrupt-resume.sdp.md:33; rebuild.sdp.md step1:125: the batch the interrupt + // replaced records the interruption, with the operator from the generation row, and schedules nothing. + w.interrupted = await waitForGeneration( + backendOf(w), + readModel, + id2(w), + ({ progress }) => + progress.lastError === "interrupted by interrupt operator", + (entry) => observe(w, entry), + ); // rebuild.online-rebuild-interrupt-resume.sdp.md:33: the interrupt lands after at least three and below 350 batches, with a tenant cursor. expect(w.interrupted.progress.batchesDone).toBeGreaterThanOrEqual( batches, @@ -656,10 +675,6 @@ bindExample( expect(w.interrupted.progress.batchesDone).toBeLessThan(350); // rebuild.online-rebuild-interrupt-resume.sdp.md:33: the cursor names a tenant. expect(tenants).toContain(w.interrupted.progress.cursor?.tenantId); - // rebuild.sdp.md:130: the exact interrupt reason is retained at the checkpoint. - expect(w.interrupted.progress.lastError).toBe( - "interrupted by interrupt operator", - ); await sleep(150); // rebuild.online-rebuild-interrupt-resume.sdp.md:33: the interrupted checkpoint stops advancing. expect((await entry(w)).progress).toEqual(w.interrupted.progress); @@ -712,7 +727,8 @@ bindExample( fence: required(w.oldFence, "the replaced fence"), }, ); - // rebuild.online-rebuild-interrupt-resume.sdp.md:35: a queued batch under the replaced fence cannot checkpoint. + // rebuild.online-rebuild-interrupt-resume.sdp.md:35: a queued batch under the replaced fence cannot checkpoint, + // and the interruption it would record is already there. expect((await entry(w)).progress).toEqual(checkpoint); // rebuild.online-rebuild-interrupt-resume.sdp.md:36: interrupt changes only fence and operator change fields. expect( @@ -728,7 +744,7 @@ bindExample( "changedBy", ]), ); - // rebuild.sdp.md:130: the interrupt replaces the fence and records the operator. + // rebuild.sdp.md:134: the interrupt replaces the fence and records the operator. expect(w.interrupted.generation).toMatchObject({ fence: required(w.oldFence, "the original fence") + 1, changedBy: "interrupt operator", diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts index 6bc1db7..5310eb9 100644 --- a/tests/simulator/rebuild.test.ts +++ b/tests/simulator/rebuild.test.ts @@ -309,8 +309,8 @@ test("convex-test: every generation entry refuses an unknown ID and every batch expect(await snapshot(t)).toStrictEqual(before); }); -// rebuild.sdp.md Design:107, the generation with no progress row. -test("convex-test: every entry that reads the progress row and every batch over a generation with none throws", async () => { +// rebuild.sdp.md Design:107, the generation with no progress row; fnInterruptGeneration:134. +test("convex-test: every entry that reads the progress row and every batch over a generation with none throws, and the interrupt reads none", async () => { const t = rebuildApp(); const id = await start(t); const { generation, progress } = await read(t, id); @@ -319,7 +319,6 @@ test("convex-test: every entry that reads the progress row and every batch over }); const message = `No progress for generation ${id}`; for (const ref of [ - internal.rebuild.interruptGeneration, internal.rebuild.resumeGeneration, internal.rebuild.rollbackGeneration, internal.rebuild.purgeGeneration, @@ -349,6 +348,16 @@ test("convex-test: every entry that reads the progress row and every batch over () => t.mutation(ref, { generationId: id, fence: generation.fence }), message, ); + expect( + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator, + }), + ).toBeNull(); + expect(await t.run((ctx) => ctx.db.get(id))).toStrictEqual({ + ...generation, + fence: generation.fence + 1, + }); }); // actor-and-scope.sdp.md fnAssertOperator:91; rebuild.sdp.md fnStartGeneration:115 through fnFillTenants:139. @@ -570,9 +579,9 @@ test("convex-test: a tenant inserted between crossed tenants is skipped by backf ); }); -// rebuild.sdp.md step1:121, verifySteps:127, purgeSteps:128, limitOutstandingBatches:148. +// rebuild.sdp.md step1:125, verifySteps:131, purgeSteps:132, limitOutstandingBatches:152. test.each(["backfill", "verify", "purge"] as const)( - "convex-test: a stale fence leaves every row and schedule unchanged in %s", + "convex-test: in %s the batch the interrupt replaced records it once and a batch of an older chain writes nothing", async (pass) => { const t = rebuildApp(); await create(t, "a"); @@ -593,7 +602,7 @@ test.each(["backfill", "verify", "purge"] as const)( const fence = (await read(t, id)).generation.fence; await t.mutation(internal.rebuild.interruptGeneration, { generationId: id, - operator, + operator: "interrupter", }); const before = await snapshot(t); const ref = @@ -602,12 +611,34 @@ test.each(["backfill", "verify", "purge"] as const)( : pass === "verify" ? internal.rebuild.verifyBatch : internal.rebuild.purgeBatch; + later(); await t.mutation(ref, { generationId: id, fence }); - expect(await snapshot(t)).toStrictEqual(before); + const recorded = await snapshot(t); + expect(recorded).toStrictEqual({ + ...before, + progress: [ + { + ...before.progress[0], + lastError: "interrupted by interrupter", + updatedAt: Date.now(), + }, + ], + }); + later(); + await t.mutation(ref, { generationId: id, fence }); + expect(await snapshot(t)).toStrictEqual(recorded); + await t.mutation(internal.rebuild.resumeGeneration, { + generationId: id, + operator, + }); + const resumed = await snapshot(t); + later(); + await t.mutation(ref, { generationId: id, fence }); + expect(await snapshot(t)).toStrictEqual(resumed); }, ); -// rebuild.sdp.md fnInterruptGeneration:130, fnResumeChain:131, fnResumeGeneration:132; rebuild.online-rebuild-interrupt-resume.sdp.md:38. +// rebuild.sdp.md fnInterruptGeneration:134, fnResumeChain:135, fnResumeGeneration:136; rebuild.online-rebuild-interrupt-resume.sdp.md:38. test("convex-test: interrupt after exactly three batches and resume at the stored size with a new fence", async () => { const t = rebuildApp(); for (let i = 0; i < 6; i++) await create(t, `d${i}`); @@ -633,11 +664,7 @@ test("convex-test: interrupt after exactly three batches and resume at the store changedAt: 1040, changedBy: "interrupted", }, - progress: { - ...before.progress, - lastError: "interrupted by interrupted", - updatedAt: 1040, - }, + progress: before.progress, }); expect(await scheduled(t)).toStrictEqual(queued); later(); @@ -668,21 +695,87 @@ test("convex-test: interrupt after exactly three batches and resume at the store expect(next.progress.cursor).not.toEqual(resumed.progress.cursor); }); -// rebuild.sdp.md fnInterruptGeneration:130, fnResumeChain:131. +// rebuild.sdp.md fnInterruptGeneration:134, step1:125; rebuild.online-rebuild-interrupt-resume.sdp.md:38. +test("convex-test: an interrupt while a batch is scheduled leaves the progress row untouched, and that batch records the interruption and schedules nothing", async () => { + const t = rebuildApp(); + for (let i = 0; i < 3; i++) await create(t, `d${i}`); + const id = await start(t, { batchSize: 1 }); + later(); + await batch(t, id); + const before = await read(t, id); + const args = { generationId: id, fence: before.generation.fence }; + await lastSchedule(t, "backfillBatch", args); + const queued = await scheduled(t); + const titles = await rows(t); + later(); + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator: " interrupter ", + }); + const interrupted = await read(t, id); + expect(interrupted).toStrictEqual({ + generation: { + ...before.generation, + fence: before.generation.fence + 1, + changedAt: Date.now(), + changedBy: "interrupter", + }, + progress: before.progress, + }); + later(); + await t.mutation(internal.rebuild.backfillBatch, args); + expect(await read(t, id)).toStrictEqual({ + generation: interrupted.generation, + progress: { + ...before.progress, + lastError: "interrupted by interrupter", + updatedAt: Date.now(), + }, + }); + expect(await rows(t)).toStrictEqual(titles); + expect(await scheduled(t)).toStrictEqual(queued); +}); + +// rebuild.sdp.md fnInterruptGeneration:134, fnResumeChain:135. test.each(states)( - "convex-test: an idle pass in %s cannot be interrupted or resumed", + "convex-test: an idle pass in %s cannot be resumed, and the interrupt decides from the state alone", async (state) => { const t = rebuildApp(); const id = await inState(t, state); - for (const [ref, word] of [ - [internal.rebuild.interruptGeneration, "interrupt"], - [internal.rebuild.resumeGeneration, "resume"], - ] as const) + await refused( + t, + () => + t.mutation(internal.rebuild.resumeGeneration, { + generationId: id, + operator, + }), + "Generation 1 of documentTitle has no batch to resume", + ); + const interrupt = () => + t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator: "interrupter", + }); + if (state === "verified" || state === "active" || state === "purged") { await refused( t, - () => t.mutation(ref, { generationId: id, operator }), - `Generation 1 of documentTitle has no batch to ${word}`, + interrupt, + "Generation 1 of documentTitle has no batch to interrupt", ); + return; + } + const before = await read(t, id); + later(); + await interrupt(); + expect(await read(t, id)).toStrictEqual({ + generation: { + ...before.generation, + fence: before.generation.fence + 1, + changedAt: Date.now(), + changedBy: "interrupter", + }, + progress: before.progress, + }); }, ); From 131b48ad9ae21f7ad3db5ec17088a94874f64d9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 11:36:35 +0200 Subject: [PATCH 151/155] build: the interrupt marks its fence The generation row gains interruptedFence, which interruptGeneration sets to the new fence and every other fence bump leaves. A stale batch records the interruption on the progress row only while interruptedFence equals the fence and the row does not already hold the note, so a batch pending across an interrupt and a resume writes nothing and one pending across two interrupts records the second once. Codegen leaves both compositions' generated files unchanged. STATE.md closes the L2-5 half of Leads 43. Co-Authored-By: Claude Opus 5.5 --- design/STATE.md | 2 +- .../application/generation-registry.sdp.md | 4 +- ...ild.online-rebuild-interrupt-resume.sdp.md | 2 +- design/specs/application/rebuild.sdp.md | 10 +-- design/specs/application/write-pause.sdp.md | 2 +- src/read-model/rebuild.ts | 14 ++-- src/read-model/tables.ts | 1 + ...ld.online-rebuild-interrupt-resume.test.ts | 7 +- tests/simulator/rebuild.test.ts | 64 ++++++++++++++++++- tests/types/rebuild.test-d.ts | 1 + 10 files changed, 89 insertions(+), 18 deletions(-) diff --git a/design/STATE.md b/design/STATE.md index a239f71..ae656a6 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -460,7 +460,7 @@ Leads 30 to 38 are lane data's of the tooling unit, as `leads-data.md` in its fo 43. From the close of round 3. Each is a gap the build of its subject takes, a test gap, a number to take again, or a failure to read again. - The history code gap: step 9 has no history or aggregate path, so the pipeline writes no history projection although the Specs pin one `applyHistoryProjection` call per command for each bound history read model. The history view's build (5d) takes it. F9's code side, `ReadModelBinding` with `AggregateProjection`, waits for OD-075. - Probe 4's 1 s timeouts under load: its component and nested files time out at the one-second execution limit on the bytes-written step. They failed in the integrator's whole-tier run at `e97db90` and passed alone there, and failed in the run on `c1bdf80` and again alone at a load of 13 to 18. In CI at `d2a8d1b` the nested file timed out at 1 s in its Given step, which writes documents of 900 KiB, so the sensitivity is not only this machine's. Two lint tests in `tests/pure/compositions.test.ts` time out at their 5 s under the same load. - - Sc L2-5 in CI at `d2a8d1b`: `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` and the run threw `OptimisticConcurrencyControlFailure` at the interrupt step. 3b's F1 counted the interrupt's reruns as designed; whether a long batch chain can starve the interrupt, and what the operator then does, has no bound yet. The local whole-tier runs of this unit passed Sc L2-5. + - Sc L2-5 in CI at `d2a8d1b`: `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` and the run threw `OptimisticConcurrencyControlFailure` at the interrupt step. 3b's F1 counted the interrupt's reruns as designed; whether a long batch chain can starve the interrupt, and what the operator then does, has no bound yet. The local whole-tier runs of this unit passed Sc L2-5. Closed by `2a44655` and `build: the interrupt marks its fence`: the interrupt reads and writes only the generation row and sets `interruptedFence`, the stale batch records the interruption while that mark equals the fence, so the interrupt conflicts with a batch only through the generation row, and the Probe 4 half of that CI failure is addressed by `4ff759d`. - The hosted examples assume that a backup archive holds every mounted component's tables (F26, F41); the first hosted run shows it. - What sol's read of the fold-in left. F38's 300 s deadline bounds when the usage polling admits a request and not when it completes, since the last usage request has its own 30 s timeout. F7 assigns the `awaitingBaseline` resolution to `finishRestoreRun` while the chain restarts after acceptance, and `fnFinishRestoreRun` handles no such resolution: paper for the restore's build (5b). F18's equality test with 1,000 versions would not catch a return to the linear search: a cost, not a correctness, check. - After the follow-up `651fcc3`, an error whose `cause` is cyclic comes back as a redacted stub rather than a redacted copy; a visited set would keep its message. diff --git a/design/specs/application/generation-registry.sdp.md b/design/specs/application/generation-registry.sdp.md index 8c968ea..8fb4bc7 100644 --- a/design/specs/application/generation-registry.sdp.md +++ b/design/specs/application/generation-registry.sdp.md @@ -50,7 +50,7 @@ The registry is two parent tables. The `generations` table records every generat - A command that maintains a read model writes the active generation and the in-flight generation if one exists; a retired generation is kept unchanged until it is purged and never written, and a rollback reopens it as `verifying` so that live commands write it again while its verify pass repairs it; a generation with `pauseRequired` is never reopened, because nothing would close its sources while live commands fold onto rows that missed the events since its retirement (D9, E-8) - [extension] Each generation is written with the projection whose version its row records, on the live path and by its batches alike, so the rows of one generation are one truth of one projection version (E-9, E-8, D8) - A query reads the active generation and nothing else; a query over a read model that has no active generation throws a plain error that names the read model, a technical failure, and never answers an empty page (D9, E-8, Law 6) -- [extension] The generation row carries the state, the projection version, the fence and who started and last changed it, and nothing a batch writes on its healthy path except the state that the batch ending a pass sets; the progress row carries the cursor, the batch size and the counts, the batch size set by `startGeneration` and never changed after, so every pass of the generation, a resumed one and its purge included, runs at it, a purge batch capped further by `limitPurgeBatch` of `spec:application.rebuild`, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) +- [extension] The generation row carries the state, the projection version, the fence, the fence the last interrupt set and who started and last changed it, and nothing a batch writes on its healthy path except the state that the batch ending a pass sets; the progress row carries the cursor, the batch size and the counts, the batch size set by `startGeneration` and never changed after, so every pass of the generation, a resumed one and its purge included, runs at it, a purge batch capped further by `limitPurgeBatch` of `spec:application.rebuild`, and a batch writes it in the same mutation as its rows, so the progress row is the checkpoint (E-40, D9, F1) - [extension] A generation spans every tenant of the deployment and the registry row names none; the tenant is named on every read-model row and marker, on every context call a batch makes, and in the batch cursor, so no read or write of tenant data runs with an absent tenant as a wildcard, and no generation can be switched for one tenant alone (E-8, D11, Law 11) - An aggregate read model keeps one marker per entity per generation with the contribution last recorded and the source versions it was computed from, and every aggregate write reads and writes the marker in the same mutation as the aggregate row; the marker's `sourceVersions` are compared like a row's, so a DTO older than the marker never adjusts the sum (D9, E-8, E-9) - [extension] A generation whose progress row is in the pass `backfill`, `verify` or `purge` has at most one scheduled batch driving it, identified by the row's fence; a restore brings the rows back at their checkpoint and drops that batch with every pending scheduled function, so the generation is stranded until `resumeChain` bumps the fence and reschedules from the saved cursor, which the restore procedure's pending-work inventory and accepted branch drive for every such generation in `building` or `verifying`, and which the operator drives with `resumeGeneration` for a purge the restore stranded, because a purge belongs to a `retired` or `aborted` generation the inventory does not read (E-42, E-8, F12, D19) @@ -64,7 +64,7 @@ The registry is two parent tables. The `generations` table records every generat Tables live in the parent's `schema.ts`, spread from the library's `readModelTables`. `streamVersionValidator` is the kernel's validator for `StreamVersion`. The registry has no `tenantId` because every generation spans all tenants; the marker table is tenant data and leads with the tenant. Every refusal below is a plain `Error` with the message given. - validatorGenerationState: `const generationStateValidator = v.union(v.literal("building"), v.literal("verifying"), v.literal("verified"), v.literal("active"), v.literal("retired"), v.literal("aborted"), v.literal("purged"))` (E-8, D9) -- tableGenerations: `generations: defineTable({ readModel: v.string(), generation: v.number(), projectionVersion: v.number(), state: generationStateValidator, pauseRequired: v.boolean(), fence: v.number(), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), switchedAt: v.optional(v.number()), retiredAt: v.optional(v.number()), retireAfter: v.optional(v.number()) })` where `changedAt` and `changedBy` are the time and the operator of the last operator entry that changed the row, which no batch writes, `switchedAt` is when the generation last became active, and `retiredAt` and `retireAfter` are set while it is `retired` and absent otherwise (E-8, E-48, D9, F13) +- tableGenerations: `generations: defineTable({ readModel: v.string(), generation: v.number(), projectionVersion: v.number(), state: generationStateValidator, pauseRequired: v.boolean(), fence: v.number(), startedAt: v.number(), startedBy: v.string(), changedAt: v.number(), changedBy: v.string(), switchedAt: v.optional(v.number()), retiredAt: v.optional(v.number()), retireAfter: v.optional(v.number()), interruptedFence: v.optional(v.number()) })` where `changedAt` and `changedBy` are the time and the operator of the last operator entry that changed the row, which no batch writes, `switchedAt` is when the generation last became active, `retiredAt` and `retireAfter` are set while it is `retired` and absent otherwise, and `interruptedFence` is the fence `interruptGeneration` last set, absent before the first interrupt, so it equals `fence` only while the interrupt made the last bump, and every other entry that adds to the fence leaves it as it is (E-8, E-48, D9, F13) - generationPauseRequired: `pauseRequired` is true on every generation of a history view, set by `startGeneration` from its rebuild target and changed by nothing, and false on every other; `switchGeneration` and `abortGeneration` reopen the source scopes such a generation closed, and `rollbackGeneration` refuses it (E-8, D9) - indexGenerationsByReadModel: `.index("by_read_model", ["readModel", "generation"])` (E-8) - indexGenerationsByReadModelUse: the next generation number at `startGeneration`, read as the last row in descending order, and the operator listing `getGenerations` (E-8, D9) diff --git a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md index aa0e108..28f8553 100644 --- a/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md +++ b/design/specs/application/rebuild.online-rebuild-interrupt-resume.sdp.md @@ -33,6 +33,6 @@ And writes {writes: "never stopped"} - The test, in a file whose own timeout is 300 s, installs the order summary's generation 1 through its first rebuild, seeds 350 orders across two tenants, 250 in the first in tenant-ID order and 100 in the second, starts generation 2 with `projectionVersion` 1 stated, a batch size of 1 and a stated operator, runs `PlaceOrder` and `CancelOrder` concurrently with the batches in both tenants, and calls `interruptGeneration` once the progress row shows at least 3 batches; it asserts that the batch the interrupt replaced then sets the progress row's `lastError` to `"interrupted by "` and the interrupt's operator, that `batchesDone` stops advancing below 350 and that the cursor names a tenant; while a chain is expected to run, a progress row whose `batchesDone` has not advanced for 30 s fails the test with the generation row and the progress row in the message; once generation 2 is `verified` it asserts that the backfill and verify completions in the function log that wrote number the progress row's `batchesDone` plus the one backfill batch that recorded the interruption, and that each pass, that batch aside, took at least the 350 seeded orders and at most the orders at `verified` plus one per tenant. - While the chain is interrupted the test sends `CancelOrder` to one of the second tenant's 100 seeded orders and to an order of the first tenant whose row generation 2 already holds, read from the table before the command; it asserts at once that the first tenant's order has its row in generation 2 `cancelled`, and, once generation 2 is `verified`, that the second tenant's order has its row in generation 2 `cancelled`, whichever pass wrote it. - After `resumeGeneration` the test asserts that the batch cursor advanced from the first tenant to the second and that, once the generation is `verified`, every order the Orders context's `list` enumerates in either tenant, read with admin access and `includeDeleted`, has a row in the new generation whose `sourceVersions` equal the order's current stream version, that no row is older than the state read through the component query, that `backfillBatch` run with admin access under the fence the interrupt replaced leaves the progress row unchanged, that the function log shows no action and no completion during the batches other than a batch, an operator entry or a live command, and that the journals, read by operation, hold the event and receipt counts the live commands produced. -- The test asserts that no command of the run ends in a failure, that no completion record of a `PlaceOrder`, a `CancelOrder` or a parent query names `generationProgress` in its `occInfo`, that no completion record of `interruptGeneration` names `generationProgress` either, since the interrupt conflicts with a batch only through the generation row and a batch that conflicts reruns once and stops on the new fence, and that the completion records whose `occInfo` names `generations` number at most the operator entries and pass ends of the run times the callers the test keeps in flight plus one for the chain, and records those reruns with their `occInfo` as a measurement, beside them the reruns of batches and of the other operator entries whose `occInfo` names `generationProgress`, each with its `occInfo`; it reads generation 2's row at the interrupt, after the resume and at `verified`, and asserts that the interrupt and the resume changed only `fence`, `changedAt` and `changedBy` and that from the resume to `verified` only `state` changed. +- The test asserts that no command of the run ends in a failure, that no completion record of a `PlaceOrder`, a `CancelOrder` or a parent query names `generationProgress` in its `occInfo`, that no completion record of `interruptGeneration` names `generationProgress` either, since the interrupt conflicts with a batch only through the generation row and a batch that conflicts reruns once and stops on the new fence, and that the completion records whose `occInfo` names `generations` number at most the operator entries and pass ends of the run times the callers the test keeps in flight plus one for the chain, and records those reruns with their `occInfo` as a measurement, beside them the reruns of batches and of the other operator entries whose `occInfo` names `generationProgress`, each with its `occInfo`; it reads generation 2's row at the interrupt, after the resume and at `verified`, and asserts that the interrupt changed only `fence`, `interruptedFence`, `changedAt` and `changedBy`, that the resume changed only `fence`, `changedAt` and `changedBy`, and that from the resume to `verified` only `state` changed. - The test switches, asserts from the function log that the list query ran again for each tenant's subscription after the switch and that each subscription's value holds the same rows, and that generation 1 is `retired`, then runs one more `PlaceOrder` and cancels one earlier order so that its summary changes, rolls back by naming generation 1, asserts that the reopened generation reaches `verified` after one verify pass with a row for the new order and the cancelled order's row `cancelled` before it becomes active, switches again, and asserts that generation 1 is active, generation 2 is `retired` and the list query ran again for each subscription. - A `convex-test` test of the same workflow runs on the fixture composition with `documentTitle` and calls each batch directly, so it interrupts after exactly 3 batches, and asserts that an interrupt issued while a batch is scheduled leaves the progress row untouched and that the scheduled batch records the interruption and schedules nothing; in one run generation 1 is installed at version 1 stated and generation 2 built at version 2 under live `CreateTwiceSummarizedDocument` writes, and it asserts that each generation's rows carry their own generation's `projectionVersion` and that a document whose `title` is the depot's `deletedTitle` has a row at version 1 and none at version 2; in a second run both generations are at version 2 and, after the switch, it deletes a document by `AmendDocument` with the depot's `deletedTitle`, rolls back by naming generation 1 and asserts that the reopened generation holds no row for it after its verify pass; and it asserts every refusal of the operator entries by its message. diff --git a/design/specs/application/rebuild.sdp.md b/design/specs/application/rebuild.sdp.md index 7ec52a5..0a87d74 100644 --- a/design/specs/application/rebuild.sdp.md +++ b/design/specs/application/rebuild.sdp.md @@ -88,11 +88,11 @@ The registry, the progress row, the per-entity marker and the gate document are - rule: [extension] Every batch, whether its source is a context's enumeration or a parent table, is driven by a self-scheduled internal mutation under one batch contract, and the rebuild uses `@convex-dev/migrations` for nothing (E-40, F17, F8) - rule: [extension] A projection version change is a new generation, never an in-place rewrite of the active one (E-8, D8) - rule: [extension] The batch bound is a field of the progress row, the read model's by `batchSizeFor`, a whole number of at least 1 and at most `rebuildBatchCeiling(readModel)`; the context's `list` page bounds the bytes a batch reads, so a batch may take fewer entities than its size, and does not change the size; `startGeneration` sets it and nothing changes it after, so every pass and every resumed pass runs at the size on the row, and a purge batch deletes at most that many rows, capped by `limitPurgeBatch(readModel)` (E-40, E-24, E-9, F1, F13) -- rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them checkpoints nothing and schedules nothing; the pending batch of a chain `interruptGeneration` replaced writes only the interruption, on the progress row (E-40, E-8, F8) +- rule: [extension] A generation has at most one live batch chain, the one whose batches carry the row's current fence; `interruptGeneration`, `resumeGeneration`, `rollbackGeneration`, `abortGeneration` and `purgeGeneration` each add one to the fence, so a batch scheduled before any of them checkpoints nothing and schedules nothing; `interruptGeneration` alone also sets `interruptedFence` to the new fence, and while it equals `fence` the stale batch writes only the interruption, on the progress row (E-40, E-8, F8) - An operator calls `startGeneration` for a read model; it inserts the generation row in state `building` with the next generation number, the projection version and fence 1, inserts the progress row in the pass `backfill` with the batch size and a `null` cursor, and schedules the first backfill batch, all in one mutation; a read model with no generation gets generation 1 this way (D9, F8, E-8) - [extension] On a deployment whose grants predate the tenant list an operator calls `fillTenants` before the first `startGeneration`; it inserts the `tenantFill` row, or reopens it, in the pass `fill` with a new fence and schedules the first fill batch; each fill batch reads the gate, takes up to `limitTenantFillBatch` further tenants from the grants after the row's cursor, inserts the tenant row of each that has none, saves the last tenant ID as the cursor and schedules the next fill batch, and the batch that finds no further tenant sets the pass to `idle` and schedules nothing (E-49, E-48, E-40, F8) - From that commit every use case that maintains the read model writes the active generation, when there is one, and the one being built through `applyProjection`, in live-created mode for a stream it created and live-updated mode otherwise (D9, E-9) -- Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building` or the pass is not `backfill`, returns when the fence differs, after recording an interruption as step 1 states, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) +- Each backfill batch runs as one mutation: it loads the generation row and the progress row, returns without writing if the state is not `building` or the pass is not `backfill`, returns when the fence differs, after recording the interruption as step 1 states while `interruptedFence` equals the fence, takes its tenant from the cursor, reads the gate, calls the source's `list` query once with `includeDeleted` and a page of the batch size from the saved cursor, applies the projection in backfill mode for each returned DTO at its current version, writes the new cursor and the counts on the progress row, and schedules the next batch, or, when the last tenant's last page is done, moves the generation to `verifying` and schedules the first verify batch (D9, D10, F1, E-40, E-49) - A batch that throws leaves the generation at its last checkpoint; a backend interruption leaves the scheduled batch pending and it runs after the restart, an internal error is retried by Convex, and a developer error or a timeout ends the chain; the operator's `interruptGeneration` ends a chain on purpose, and `resumeGeneration` schedules a batch from the saved cursor with a new fence, at the batch size on the progress row; re-processing an entity is a no-op because its row is no longer older; a restore is the one interruption that leaves no batch pending, so `spec:application.restore` inventories every generation in `building` or `verifying` and its accepted branch resumes each one the operator did not abort through the same `resumeChain` (D9, F9, F1, F12, E-42, Sc L2-5) - Each verify batch enumerates the sources again from the progress row's cursor, which the end of the backfill reset to `null`, the first tenant, and calls `applyProjection` in backfill mode for every DTO, so a missing row is inserted, an older one updated, and the row of a DTO that projects to `null` deleted; each such repair is counted as a miss; when the last tenant's last page is done the generation moves to `verified` and no batch is scheduled (D9, E-8, Sc L2-5) - `switchGeneration` sets the verified generation `active` and, when the read model has an active generation, sets that one `retired` with `retiredAt` and `retireAfter`, in one mutation; every query reads the active generation from the registry, so every subscription on the read model re-runs (D9, F7, F1) @@ -122,7 +122,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - fnVerifyBatch: `export const verifyBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D9) - fnPurgeBatch: `export const purgeBatch = internalMutation({ args: { generationId: v.id("generations"), fence: v.number() }, returns: v.null(), handler })` (D19, E-40) - fnEnumerateCall: `ctx.runQuery(target.list, { tenantId, includeDeleted: true, paginationOpts: { numItems: batchSize, cursor: pageCursor } })` returning the `paginator` page `{ page, isDone, continueCursor }` of DTOs with their stream versions, one call per batch; the query caps `numItems` at its byte-derived bound, so a batch never reads more than 8 MiB of stream rows however `batchSize` was set (D2, D10, F15, S4, E-24) -- step1: `backfillBatch` loads the generation row by ID and its progress row by `by_generation`, and returns without writing when the row is absent, `state` is not `building` or `pass` is not `backfill`; when `fence` differs from its argument it checkpoints nothing, schedules nothing and returns, and when its argument is one below `fence`, which after `interruptGeneration` makes it the pending batch of the chain the interrupt replaced, it first patches the progress row's `lastError` to `"interrupted by " + changedBy`, the generation row's, and `updatedAt`, unless `lastError` already says so; an interrupt therefore conflicts with a batch only through the generation row, and a batch that conflicts reruns once and stops on the new fence (E-40, E-48, D9, Sc L2-5) +- step1: `backfillBatch` loads the generation row by ID and its progress row by `by_generation`, and returns without writing when the row is absent, `state` is not `building` or `pass` is not `backfill`; when `fence` differs from its argument it checkpoints nothing, schedules nothing and returns, and when `interruptedFence` equals `fence`, so the interrupt made the last bump, it first patches the progress row's `lastError` to `"interrupted by " + changedBy`, the generation row's, and `updatedAt`, unless `lastError` already says so; a batch pending across an interrupt and a resume therefore writes nothing, one pending across two interrupts records the second once, an interrupt conflicts with a batch only through the generation row, and a batch that conflicts reruns once and stops on the new fence (E-40, E-48, D9, Sc L2-5) - step2: it takes its tenant: the cursor's `tenantId`, or `nextTenant(ctx, null)` when the cursor is `null`; when the tenant list is empty the batch has no tenant, reads the gate as step 3 states, and goes to step 6 with the pass done (E-49, Law 11) - step3: it calls the write pause's `gateAllows(ctx, ["all", "tenant:" + tenantId], generationId)`, with `["all"]` alone when it has no tenant, and when the answer is not allowed it parks: it patches the progress row's `lastError` to `"write paused for " + scopeKey + ": " + reason` and `updatedAt`, schedules nothing and returns, so the generation waits for `resumeGeneration` after the gate reopens; no batch calls `assertWritable`, because a batch parks its generation rather than failing its scheduled mutation with a thrown refusal (E-8, E-42, D9) - step4: it calls `list` once as `fnEnumerateCall` states, with the cursor's `pageCursor` (D10, E-40) @@ -131,7 +131,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - verifySteps: `verifyBatch` runs the same six steps with `state` `verifying` and `pass` `verify`, adding one to `misses` for `inserted`, `updated` or `deleted` and one to `rowsSkipped` otherwise; a pass that is not done schedules `refs.verifyBatch`; a pass that is done patches the generation row to `state: "verified"` and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, the counts, `batchesDone` plus 1 and `updatedAt`, and schedules nothing (D9, E-8, Sc L2-5) - purgeSteps: `purgeBatch` runs steps 1 to 3 with `state` `retired` or `aborted` and `pass` `purge`; it then reads `ctx.db.query(readModel.table).withIndex("by_key", (q) => q.eq("tenantId", tenantId).eq("generation", n)).take(Math.min(batchSize, limitPurgeBatch(readModel)))` with `batchSize` the progress row's and `readModel` the generation's read model, whose cap `limitPurgeBatch` declares, deletes each row and adds their number to `rowsPurged`; when it deleted that many rows the cursor stays on the tenant, and otherwise it moves to `nextTenant(ctx, tenantId)`, a purge cursor being `{ tenantId, pageCursor: null }`; a pass that is not done checkpoints, patching the progress row with the cursor, `rowsPurged`, `batchesDone` plus 1, no `lastError` and `updatedAt`, and schedules `refs.purgeBatch`; a pass that is done patches the generation row to `state: "purged"`, no `retiredAt` and no `retireAfter`, and nothing else on it, and the progress row to `pass: "idle"`, `cursor: null`, `rowsPurged`, `batchesDone` plus 1 and `updatedAt` (D19, E-40, E-49, Law 11) - purgeDriver: the purge batch is driven by the same self-scheduled internal mutation as the other batches (E-40, F17, Probe 6) -- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, reading the generation row and nothing else, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `state` is `verified`, `active` or `purged`, in which no pass runs; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, and scheduling nothing; a `retired` or `aborted` generation runs a pass only during a purge, which its row does not tell, so the interrupt accepts either and the new fence changes nothing when no batch runs; it neither reads nor patches the progress row, so no checkpoint conflicts with it: the chain's pending batch records the interruption as step 1 states and the generation stays at its checkpoint, and a parked or stranded chain, which has no pending batch, keeps its progress row as it was (E-40, E-48, D9, Sc L2-5) +- fnInterruptGeneration: `export const interruptGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, reading the generation row and nothing else, throwing `"Generation " + n + " of " + name + " has no batch to interrupt"` when `state` is `verified`, `active` or `purged`, in which no pass runs; otherwise adding 1 to the fence, setting `interruptedFence` to the new fence, setting `changedAt` and `changedBy`, and scheduling nothing; a `retired` or `aborted` generation runs a pass only during a purge, which its row does not tell, so the interrupt accepts either and the new fence changes nothing when no batch runs; it neither reads nor patches the progress row, so no checkpoint conflicts with it: the chain's pending batch finds `interruptedFence` equal to the fence and records the interruption as step 1 states, the generation stays at its checkpoint, and a parked or stranded chain, which has no pending batch, keeps its progress row as it was (E-40, E-48, D9, Sc L2-5) - fnResumeChain: `resumeChain(ctx: MutationCtx, config: RebuildConfig, generationId: Id<"generations">, operator: string): Promise` throwing `"Generation " + n + " of " + name + " has no batch to resume"` when `pass` is `idle`; otherwise adding 1 to the fence, setting `changedAt` and `changedBy`, patching the progress row to no `lastError` and `updatedAt` and leaving its `batchSize`, cursor and counts as they are, and scheduling the batch of the pass, `refs.backfillBatch`, `refs.verifyBatch` or `refs.purgeBatch`, with the new fence, all in the caller's mutation; called by `resumeGeneration` and by the restore's `finishRestoreRun` for every generation the backup archive stranded (D9, F8, E-40, E-42) - fnResumeGeneration: `export const resumeGeneration = internalMutation({ args: { generationId: v.id("generations"), operator: v.string() }, returns: v.null(), handler })` `assertOperator` and then `resumeChain`; it takes no batch size, so a resumed pass runs at the progress row's `batchSize`, a purge capped by `limitPurgeBatch`; for a generation in any state whose progress row names a pass, so it continues a backfill or verify pass of a generation in `building` or `verifying` and a started purge of a `retired` or `aborted` one, which is how an operator continues a purge a restore stranded (D9, Sc L2-5, E-40, E-48) - fnSwitchGeneration: `export const switchGeneration = internalMutation({ args: { generationId: v.id("generations"), rollbackPeriodMs: v.optional(v.number()), operator: v.string() }, returns: v.null(), handler })` after `assertOperator`, throwing `"Generation " + n + " of " + name + " is " + state + "; switchGeneration needs verified"` unless the state is `verified`, and `"The rollback period must be between 0 and 2592000000 ms, and " + rollbackPeriodMs + " is not"` unless it is absent or a whole number in that range; then, with `now` the mutation's `Date.now()`, it reads the read model's active generation by `by_read_model_state` and, when there is one, patches it to `state: "retired"`, `retiredAt: now`, `retireAfter: now + (rollbackPeriodMs ?? 604800000)`, `changedAt` and `changedBy`, patches the verified generation to `state: "active"`, `switchedAt: now`, no `retiredAt`, no `retireAfter`, `changedAt` and `changedBy`, and, for a generation with `pauseRequired`, calls the write pause's `resumeScope` for every closed entry that names it (D9, E-8, E-48, F1) @@ -149,7 +149,7 @@ The enumeration is the context's `list` query with `includeDeleted`, the one rea - occRace: a live command that read a row before the batch wrote it, or the batch that read the source before a command changed it, is the loser of an optimistic-concurrency conflict and re-runs; the re-run reads the newer row or the newer source, and the version compare then skips the older write; the batch's read set is the index range of the page it enumerated, so a live write to any stream in that range re-runs the batch; a batch reads the generation row and never writes it outside the end of a pass, and it writes the progress row, which no command reads, so a batch's checkpoint re-runs no command (F1, D9, Probe 6, E-40) - limitBatchSize: `rebuildBatchCeiling(readModel) = min(100, floor(4194304 / (2 * rowBudgetBytes)))` entities per backfill or verify batch of a per-entity read model and events per batch of a history view, a history projection that declares no `rowBudgetBytes` counting 16 KiB, 100 at the default 16 KiB row budget and 32 at the 64 KiB ceiling of `limitRowSize`, this design's own bound under E-40 and F13 until Probe 6; the context's `list` caps the page further at `limitListPage(budgetBytes)`, so a batch reads 16 stream rows at the default 256 KiB stream budget and the full 100 only for a stream type whose budget is 40 KiB or less; the worst case of one batch is one `list` call of at most 8 MiB of stream rows read inside the context, plus in the parent the gate document, one tenant row, the generation row, the progress row and one point read per entity of at most `rowBudgetBytes`, and one write per entity plus the progress row, so a batch reads under 12 MiB and writes under 4 MiB even if limits add up across the boundary, and runs under 1 second (E-40, E-24, E-9, F13, Probe 4) - limitPurgeBatch: `limitPurgeBatch(readModel: ReadModel | HistoryProjection): number`, `min(500, floor(4194304 / readModel.rowBudgetBytes))` rows, a history projection that declares no `rowBudgetBytes` counting 16 KiB, the cap on the rows a purge batch deletes, which deletes the progress row's `batchSize` when that is smaller; 256 at the default row budget and 64 at the ceiling, one index range of one tenant, so a batch reads at most `limitPurgeBatch` rows of at most `rowBudgetBytes` and the 61 bytes of a stored document's system fields each, 4,209,920 bytes at the default row budget and 4,198,208 at the ceiling (E-40, F13, D19) -- limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch checkpoints nothing and schedules nothing, and writes at most the interruption step 1 records (E-40, F13) +- limitOutstandingBatches: one live batch chain per generation, enforced by the fence; a stale batch checkpoints nothing and schedules nothing, and writes at most the interruption step 1 records while `interruptedFence` equals the fence (E-40, F13) - limitBuildingGenerations: at most one generation in `building`, `verifying` or `verified` per read model at a time, so a command writes at most two generations, the active one and that one (E-8, E-9) - limitRollbackPeriod: `[extension]` 7 days by default, stored as `retireAfter` on the retired row; the retired generation is kept and not written, and a rollback at or after `retireAfter` is refused (E-8, D9) - limitGenerationsListed: 20 generations per `getGenerations` answer, the newest first, a library constant (E-8, F13) diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index c4ba4da..7d60c78 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -45,7 +45,7 @@ The write pause is the maintenance gate every writer in a scope reads inside its - A deployment with no gate document is open: the gate read never writes, and the first close inserts the document (E-8, D9) - [extension] A rebuild that needs a consistent cut closes the sources of the paused view, one `source:` scope per context and stream type its bindings name, so every writer of those streams is held whether or not it maintains the view; the closed entries name the generation, and `abortGeneration` and the switch's resume reopen every entry that names it (E-8, E-38, D9, Sc L2-6) - [extension] The rebuild a scope was closed for is the one background writer allowed through it: a gate read that states a generation passes a closed entry only when the entry names a generation and names that one, so an entry with no generation admits no one, and a gate read that states no generation passes no closed entry (E-8, D9) -- Background writers are fenced, not drained: a batch scheduled before the close reads the closed gate when it starts and refuses, a batch running when the close commits loses the optimistic-concurrency race on the gate document and re-runs into the refusal, a batch that committed first wrote before the cut, and a batch of a superseded chain returns without writing because the generation row's fence differs from its argument (D9, E-8, F1) +- Background writers are fenced, not drained: a batch scheduled before the close reads the closed gate when it starts and refuses, a batch running when the close commits loses the optimistic-concurrency race on the gate document and re-runs into the refusal, a batch that committed first wrote before the cut, and a batch of a superseded chain checkpoints nothing and schedules nothing because the generation row's fence differs from its argument, writing at most the interruption on the progress row while the row's `interruptedFence` equals its fence (D9, E-8, F1) - The gate has one `closed` state per scope and no draining state, drain interval or confirmation step, because the serialization above already gives the consistent cut and a second state would only add an operator duty (E-8, F1) - `startGeneration` closes one entry per source of a paused view and records its own generation on each in the mutation that inserts the generation row, so every entry that names a generation names a row that exists (D9, E-8) - [extension] The operator entries are `closeGate` and `resumeGate` for the scope `all` and for a tenant scope, and `getGate` and `getGateAudit` to read; each follows the operator entry contract of `spec:command.actor-and-scope`, and a source scope is closed and reopened only by the rebuild (E-8, E-48, D19) diff --git a/src/read-model/rebuild.ts b/src/read-model/rebuild.ts index c89bfa0..01bc07c 100644 --- a/src/read-model/rebuild.ts +++ b/src/read-model/rebuild.ts @@ -229,8 +229,8 @@ export async function startGeneration( }); return generationId; } -// Reads and writes the generation row alone, so a batch's checkpoint never conflicts with it; the batch -// the new fence replaces records the interruption on the progress row. +// Reads and writes the generation row alone, so a batch's checkpoint never conflicts with it; it marks +// the new fence as its own, and the batch that fence replaces records the interruption on the progress row. export async function interruptGeneration( parentCtx: GenericMutationCtx, _config: RebuildConfig, @@ -247,6 +247,7 @@ export async function interruptGeneration( throw new Error(`${description(generation)} has no batch to interrupt`); await ctx.db.patch(generation._id, { fence: generation.fence + 1, + interruptedFence: generation.fence + 1, changedAt: Date.now(), changedBy: operator, }); @@ -481,10 +482,13 @@ async function generationBatch( : generation.state === "retired" || generation.state === "aborted"; if (!stateMatches || progress.pass !== pass) return null; if (generation.fence !== args.fence) { - // The batch one fence below is the pending batch of the chain the interrupt replaced: it records - // the interruption once; a batch of an older chain writes nothing. + // While the interrupt made the last fence bump, a stale batch records it once; after any other bump + // a stale batch writes nothing. const lastError = `interrupted by ${generation.changedBy}`; - if (args.fence === generation.fence - 1 && progress.lastError !== lastError) + if ( + generation.interruptedFence === generation.fence && + progress.lastError !== lastError + ) await ctx.db.patch(progress._id, { lastError, updatedAt: Date.now() }); return null; } diff --git a/src/read-model/tables.ts b/src/read-model/tables.ts index f243660..497a618 100644 --- a/src/read-model/tables.ts +++ b/src/read-model/tables.ts @@ -50,6 +50,7 @@ export const readModelTables = { switchedAt: v.optional(v.number()), retiredAt: v.optional(v.number()), retireAfter: v.optional(v.number()), + interruptedFence: v.optional(v.number()), }) .index("by_read_model", ["readModel", "generation"]) .index("by_read_model_state", ["readModel", "state"]), diff --git a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts index a9a7dae..a441948 100644 --- a/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts +++ b/tests/native/rebuild.online-rebuild-interrupt-resume.test.ts @@ -730,23 +730,26 @@ bindExample( // rebuild.online-rebuild-interrupt-resume.sdp.md:35: a queued batch under the replaced fence cannot checkpoint, // and the interruption it would record is already there. expect((await entry(w)).progress).toEqual(checkpoint); - // rebuild.online-rebuild-interrupt-resume.sdp.md:36: interrupt changes only fence and operator change fields. + // rebuild.online-rebuild-interrupt-resume.sdp.md:36: interrupt changes only the fence, its mark and operator change fields. expect( without(w.interrupted.generation, [ "fence", + "interruptedFence", "changedAt", "changedBy", ]), ).toEqual( without(required(w.initial, "the start row").generation, [ "fence", + "interruptedFence", "changedAt", "changedBy", ]), ); - // rebuild.sdp.md:134: the interrupt replaces the fence and records the operator. + // rebuild.sdp.md:134: the interrupt replaces the fence, marks it as its own and records the operator. expect(w.interrupted.generation).toMatchObject({ fence: required(w.oldFence, "the original fence") + 1, + interruptedFence: required(w.oldFence, "the original fence") + 1, changedBy: "interrupt operator", }); await backendOf(w).admin.run( diff --git a/tests/simulator/rebuild.test.ts b/tests/simulator/rebuild.test.ts index 5310eb9..1e28998 100644 --- a/tests/simulator/rebuild.test.ts +++ b/tests/simulator/rebuild.test.ts @@ -357,6 +357,7 @@ test("convex-test: every entry that reads the progress row and every batch over expect(await t.run((ctx) => ctx.db.get(id))).toStrictEqual({ ...generation, fence: generation.fence + 1, + interruptedFence: generation.fence + 1, }); }); @@ -581,7 +582,7 @@ test("convex-test: a tenant inserted between crossed tenants is skipped by backf // rebuild.sdp.md step1:125, verifySteps:131, purgeSteps:132, limitOutstandingBatches:152. test.each(["backfill", "verify", "purge"] as const)( - "convex-test: in %s the batch the interrupt replaced records it once and a batch of an older chain writes nothing", + "convex-test: in %s the batch the interrupt replaced records it once, and after a resume writes nothing", async (pass) => { const t = rebuildApp(); await create(t, "a"); @@ -661,6 +662,7 @@ test("convex-test: interrupt after exactly three batches and resume at the store generation: { ...before.generation, fence: 2, + interruptedFence: 2, changedAt: 1040, changedBy: "interrupted", }, @@ -677,6 +679,7 @@ test("convex-test: interrupt after exactly three batches and resume at the store generation: { ...before.generation, fence: 3, + interruptedFence: 2, changedAt: 1050, changedBy: "resumed", }, @@ -717,6 +720,7 @@ test("convex-test: an interrupt while a batch is scheduled leaves the progress r generation: { ...before.generation, fence: before.generation.fence + 1, + interruptedFence: before.generation.fence + 1, changedAt: Date.now(), changedBy: "interrupter", }, @@ -736,6 +740,62 @@ test("convex-test: an interrupt while a batch is scheduled leaves the progress r expect(await scheduled(t)).toStrictEqual(queued); }); +// rebuild.sdp.md step1:125, fnInterruptGeneration:134, fnResumeChain:135; generation-registry.sdp.md tableGenerations. +test("convex-test: a batch pending across an interrupt and a resume, or across a resume alone, writes nothing", async () => { + const t = rebuildApp(); + for (let i = 0; i < 3; i++) await create(t, `d${i}`); + const id = await start(t, { batchSize: 1 }); + await batch(t, id); + for (const entries of [ + [internal.rebuild.interruptGeneration, internal.rebuild.resumeGeneration], + [internal.rebuild.resumeGeneration], + ]) { + const pending = { + generationId: id, + fence: (await read(t, id)).generation.fence, + }; + for (const entry of entries) + await t.mutation(entry, { generationId: id, operator }); + const before = await snapshot(t); + later(); + await t.mutation(internal.rebuild.backfillBatch, pending); + expect(await snapshot(t)).toStrictEqual(before); + } +}); + +// rebuild.sdp.md step1:125, fnInterruptGeneration:134; generation-registry.sdp.md tableGenerations. +test("convex-test: a batch pending across two interrupts writes the note of the second once", async () => { + const t = rebuildApp(); + for (let i = 0; i < 3; i++) await create(t, `d${i}`); + const id = await start(t, { batchSize: 1 }); + await batch(t, id); + const before = await read(t, id); + const pending = { generationId: id, fence: before.generation.fence }; + const queued = await scheduled(t); + for (const stated of ["first", "second"]) + await t.mutation(internal.rebuild.interruptGeneration, { + generationId: id, + operator: stated, + }); + expect((await read(t, id)).generation).toMatchObject({ + fence: pending.fence + 2, + interruptedFence: pending.fence + 2, + changedBy: "second", + }); + later(); + await t.mutation(internal.rebuild.backfillBatch, pending); + const recorded = await snapshot(t); + expect((await read(t, id)).progress).toStrictEqual({ + ...before.progress, + lastError: "interrupted by second", + updatedAt: Date.now(), + }); + expect(recorded.scheduled).toStrictEqual(queued); + later(); + await t.mutation(internal.rebuild.backfillBatch, pending); + expect(await snapshot(t)).toStrictEqual(recorded); +}); + // rebuild.sdp.md fnInterruptGeneration:134, fnResumeChain:135. test.each(states)( "convex-test: an idle pass in %s cannot be resumed, and the interrupt decides from the state alone", @@ -771,6 +831,7 @@ test.each(states)( generation: { ...before.generation, fence: before.generation.fence + 1, + interruptedFence: before.generation.fence + 1, changedAt: Date.now(), changedBy: "interrupter", }, @@ -2332,6 +2393,7 @@ test("convex-test: resuming verify preserves its counts and cursor and schedules generation: { ...before.generation, fence: 3, + interruptedFence: 2, changedAt: 1010, changedBy: "resume verify", }, diff --git a/tests/types/rebuild.test-d.ts b/tests/types/rebuild.test-d.ts index ed5901a..eea8313 100644 --- a/tests/types/rebuild.test-d.ts +++ b/tests/types/rebuild.test-d.ts @@ -45,6 +45,7 @@ type Generation = { switchedAt?: number; retiredAt?: number; retireAfter?: number; + interruptedFence?: number; }; type Progress = { _id: Id<"generationProgress">; From 9e580193124546f7ccbda51cf686acf5811ecd9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 11:45:40 +0200 Subject: [PATCH 152/155] design: the close of round 3, CI green after the two fixes, the fence writes no read-model row Co-Authored-By: Claude Opus 5.5 --- CONTEXT.md | 2 +- design/STATE.md | 9 +++++---- design/specs/platform/vocabulary.sdp.md | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 54292c7..a877d38 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -221,7 +221,7 @@ Deleting a retired generation's rows. _Avoid_: cleanup, garbage collection, drop **Fence**: -The number a generation's batches carry so that a batch from before an interruption writes nothing. +The number a generation's batches carry so that a batch from before an interruption writes no read-model row and schedules nothing. _Avoid_: epoch, token, lease **Progress row**: diff --git a/design/STATE.md b/design/STATE.md index ae656a6..77aea81 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -26,7 +26,7 @@ advisor: convex=37, domain=45, operator=34, product=39 **Round 2, the tooling unit, round 3's unit 3b, sitting two and the close of round 3 are on `feature/round-2`, not on `main`.** `main` is still at `592f1dc`, the close of round 1. The close of round 3 did the rest of the mini roadmap (4f, the hosted tier, 4c, 4a and 4d) and ends the branch: `feature/round-2` goes to `main` as one pull request, which the main thread opens at this close for the owner's review and merge. The owner's words at the opening of the close: "forks have been resolved and mini roadmap unlocked. please complete the work to wrap up the feature branch and PR. use Opus, GPT 6 Astra and 6.1 Sol and orchestrate from the main thread". The main thread was Opus 5.5; no Fable agent and no advisor ran. Its head is this close's commit, on top of the fold-in's follow-up `651fcc3`. Tactical decision 42 holds the branch and its one pull request. -CI passed on the branch's pushes in round 2 (`1b92bd8`, `a5d339d`) and in 3b (`314c265`, `2283083`, `de285bb`, and its close `840b32c` in 7 m 12 s). It failed on the push of sitting two, `d2a8d1b` (run 37105650743): two native files of 166, Probe 4's nested-call example and Sc L2-5's `rebuild.online-rebuild-interrupt-resume`, whose `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` (`OptimisticConcurrencyControlFailure`), so acceptance read 31 passed, 1 failed and 6 missing (Leads 43). Nothing was pushed after it until this close. The local whole-tier runs of this unit ran under load and are no proof of the whole tier: CI's run on the push of this close, and on the pull request, is. +CI passed on the branch's pushes in round 2 (`1b92bd8`, `a5d339d`) and in 3b (`314c265`, `2283083`, `de285bb`, and its close `840b32c` in 7 m 12 s). It failed on the push of sitting two, `d2a8d1b` (run 37105650743): two native files of 166, Probe 4's nested-call example and Sc L2-5's `rebuild.online-rebuild-interrupt-resume`, whose `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` (`OptimisticConcurrencyControlFailure`), so acceptance read 31 passed, 1 failed and 6 missing (Leads 43). Nothing was pushed after it until this close. CI on the close's push, `1b3ff58` (run 37112192453), failed one file of 175, Probe 4's nested file on its 1 s limit, and passed L2-5. Two fixes followed before the pull request, each read by sol: Probe 4's documents hold an ASCII string in place of bytes, so the step reaches the bytes-written limit in about 0.06 s where it took 0.44 s (`4ff759d`); and `interruptGeneration` reads and writes the generation row only, marking `interruptedFence`, while a batch that finds its fence stale records the interruption on the progress row, so the interrupt no longer conflicts with the batches on `generationProgress` (`2a44655`, `131b48a`). CI on `131b48a` (run 37113652222, 7 m 17 s) passed: `npm test` 69 files; the whole tier 175 files, 174 passed and 1 skipped, 1,201 tests passed and 5 skipped; acceptance 34 passed, 0 failed, 0 absent, 4 missing, "run passed · tree clean". The local whole-tier runs of this unit ran under load and are no proof of the whole tier; CI's runs are. The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Slice S0 reached it through a pull request; from S1 on the session fast-forwarded it once CI was green, on the owner's mandate of 2026-10-01, until the merge of slice S2 and the platform review unit on 2026-10-02, when the owner said: "Please merge to main and push. This is the last change we will do without formal PRs since things are starting to shape up." Round 1's four lanes then reached it through pull requests #6 to #9 (decision 32). The platform review as it was received is in commit `70c15bc`, with paths of the owner's machine in its links; its files left the tree at that unit's close, and the owner said the deleted review may stay in the history. @@ -46,7 +46,7 @@ The close of round 3, `application-platform-round-3-close`, on 2026-10-03 from a - The review, over the branch since `840b32c`, began on partial heads so that it overlapped the hosted lane. Sol at medium read every line in two jobs, each a second time after the hosted merge: the Claude-written code (`job-mus3zboo-ae840d1a`, 2.8 and 4.9 min) and the paper (`job-mus3zbpu-c59dd380`, 4.9 and 4.5 min). An Opus agent read the GPT-written code, the sweep and the audit proof, in 10.1 min at `7603ccf`. `gpt-6-astra` at xhigh (`job-mus4kqxq-cdb3cd83`, 10.7 min, frozen `e97db90`) reviewed behavior: three findings, each also found by a second reviewer without the other (the redaction of `error.name` and of `AggregateError.errors` by sol's code read, the sweep's byte bound by the Opus read, which astra showed natively with 60 receipts of about 449 KB that a limit of 1,000 could not sweep); 17 mutations, 14 caught; both Sc ALL-1 verifiers and the undeclared-table example passing at the native backend tier. Every finding is ruled in `foldin-list.md`, F1 to F43. - The fold-in, one Opus agent, 29.0 min, six commits `7c70057` to `804aa51`, F1 to F43: among them the sweep's refusal of a `now` later than the mutation's time and its read bounded by 8 MiB as well as 1,000 rows, `fnSweepNext` on paper, the diagnostic's cut by binary search, an async sink counted as lost, the redaction of an error's `name`, `errors` and `cause`, "backup archive" in every Spec, and the hosted examples compared over every mounted component. `npm test` 69 files, 1,046 passed; both Sc ALL-1 verifiers and the two Probe 7 files pass natively. The main thread applied F27's sentence to the local in-place example (`73ffe6d`, its native test passing), OD-155 `decided` as the owner's and OD-036 `waiting` (`c1bdf80`), and F5's ledger note. - Sol's read of the fold-in (`job-mus5oy5q-90d6f6ac`, 4.2 min, frozen `c1bdf80`): 38 of 43 items applied as ruled and five otherwise (F5, F7, F13, F23, F38); F5 (no Spec cites a register ID) and F23 (the module keys are virtual, as a comment says) accepted, and F7, F38 and F18's test went to Leads. The main thread took one follow-up past decision 19, because a key-leak path is the hosted design's first invariant: the redaction walk returns a redacted stub past its depth bound and walks plain objects and an error's own properties, and the sweep's byte bound is asserted as the literal 8 MiB (`651fcc3`, a pure test per path). The unit stopped there. -- The whole native tier on the final code, `c1bdf80`, in a frozen copy: 175 files, 1,195 passed, 2 failed and 5 skipped in 391.7 s, at a load average of 1.92 before and 22.38 after, with this unit's agents and another project's vitest run on the machine. The two failures are Probe 4's component and nested files, the bytes-written step timing out at 1 s; rerun alone at a load of 13 to 18 they failed the same way, and at `e97db90`, at a lower load, both had passed alone (Leads 43). `npm run acceptance` on that record: 34 passed, 0 failed, 0 absent, 4 missing, "tree clean", "run failed" only for Probe 4. CI's run on the push is the proof of the whole tier. +- The whole native tier on the final code, `c1bdf80`, in a frozen copy: 175 files, 1,195 passed, 2 failed and 5 skipped in 391.7 s, at a load average of 1.92 before and 22.38 after, with this unit's agents and another project's vitest run on the machine. The two failures are Probe 4's component and nested files, the bytes-written step timing out at 1 s; rerun alone at a load of 13 to 18 they failed the same way, and at `e97db90`, at a lower load, both had passed alone (Leads 43). `npm run acceptance` on that record: 34 passed, 0 failed, 0 absent, 4 missing, "tree clean", "run failed" only for Probe 4. CI's run on the push is the proof of the whole tier: after the two fixes, CI on `131b48a` passed every file. - The ledger and the register: S4's open findings went from 15 to 4 and S5's from 19 to 16; OD-149 and OD-155 are `decided` and OD-036 `waiting`. Sitting two, round 3's unit 3d, on 2026-10-03 from about 08:00 to 09:30, on the ThinkPad, on `feature/round-2` from `840b32c`. The owner brought a table of recommended resolutions for every row and said "Please help me resolve all these forks, terminology to unlock the mini roadmap and finalization of the current feature branch." The main thread first prepared a rulings page and five lanes, and the owner stopped it: "this is excessive", then "I suggest you make recommendation for each ruling. And get feedback from 1x Opus 5.5 hxigh, 1x GPT 6 Astra with `gpt@libar-codex`. And rule." Decision 50 holds the method. What it did, at the tier named: @@ -459,12 +459,13 @@ Leads 30 to 38 are lane data's of the tooling unit, as `leads-data.md` in its fo - The words without a ruling are the Owner queue's language rows. 43. From the close of round 3. Each is a gap the build of its subject takes, a test gap, a number to take again, or a failure to read again. - The history code gap: step 9 has no history or aggregate path, so the pipeline writes no history projection although the Specs pin one `applyHistoryProjection` call per command for each bound history read model. The history view's build (5d) takes it. F9's code side, `ReadModelBinding` with `AggregateProjection`, waits for OD-075. - - Probe 4's 1 s timeouts under load: its component and nested files time out at the one-second execution limit on the bytes-written step. They failed in the integrator's whole-tier run at `e97db90` and passed alone there, and failed in the run on `c1bdf80` and again alone at a load of 13 to 18. In CI at `d2a8d1b` the nested file timed out at 1 s in its Given step, which writes documents of 900 KiB, so the sensitivity is not only this machine's. Two lint tests in `tests/pure/compositions.test.ts` time out at their 5 s under the same load. - - Sc L2-5 in CI at `d2a8d1b`: `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` and the run threw `OptimisticConcurrencyControlFailure` at the interrupt step. 3b's F1 counted the interrupt's reruns as designed; whether a long batch chain can starve the interrupt, and what the operator then does, has no bound yet. The local whole-tier runs of this unit passed Sc L2-5. Closed by `2a44655` and `build: the interrupt marks its fence`: the interrupt reads and writes only the generation row and sets `interruptedFence`, the stale batch records the interruption while that mark equals the fence, so the interrupt conflicts with a batch only through the generation row, and the Probe 4 half of that CI failure is addressed by `4ff759d`. + - Probe 4's 1 s timeouts under load: its component and nested files time out at the one-second execution limit on the bytes-written step. They failed in the integrator's whole-tier run at `e97db90` and passed alone there, and failed in the run on `c1bdf80` and again alone at a load of 13 to 18. In CI at `d2a8d1b` the nested file timed out at 1 s in its Given step, which writes documents of 900 KiB, so the sensitivity was not only this machine's. Addressed by `4ff759d`: the documents hold an ASCII string in place of bytes, and the combined write takes about 0.06 s where it took 0.44 s; CI on `131b48a` passed both files. What stays: the local backend's completion time fell with the nested call's own writes, which suggests the 1 s budget covers the call tree, against F13's `limitQueryMutationTimeout`, and a probe settles it; and the component-call file checks combined row counts, so routing its writes through the nested call instead would pass it (`tests/native/shared-limits.ts:230`, sol's read of the fixes). Two lint tests in `tests/pure/compositions.test.ts` time out at their 5 s under the same load. + - Sc L2-5 in CI at `d2a8d1b`: `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` and the run threw `OptimisticConcurrencyControlFailure` at the interrupt step. 3b's F1 counted the interrupt's reruns as designed; whether a long batch chain can starve the interrupt, and what the operator then does, has no bound yet. The local whole-tier runs of this unit passed Sc L2-5. Closed by `2a44655` and `131b48a`: the interrupt reads and writes only the generation row and sets `interruptedFence`, the stale batch records the interruption while that mark equals the fence, so the interrupt conflicts with a batch only through the generation row, and the Probe 4 half of that CI failure is addressed by `4ff759d`. - The hosted examples assume that a backup archive holds every mounted component's tables (F26, F41); the first hosted run shows it. - What sol's read of the fold-in left. F38's 300 s deadline bounds when the usage polling admits a request and not when it completes, since the last usage request has its own 30 s timeout. F7 assigns the `awaitingBaseline` resolution to `finishRestoreRun` while the chain restarts after acceptance, and `fnFinishRestoreRun` handles no such resolution: paper for the restore's build (5b). F18's equality test with 1,000 versions would not catch a return to the linear search: a cost, not a correctness, check. - After the follow-up `651fcc3`, an error whose `cause` is cyclic comes back as a redacted stub rather than a redacted copy; a visited set would keep its message. - OD-093's provisional reading quotes the old E-9 text, and the register tool does not patch that field. + - The interrupt accepts a `retired` or `aborted` generation, because its row does not show whether a purge's batches run; a fence bump with nothing running is harmless, and a purge marker on the row would tell the two apart if an interrupt of a purge must ever be refused. The reports behind leads 1 to 3 and 6 are outside the repository, because they quote a private repository. The project notes file named in `AGENTS.md` lists them, with the reports, briefs and rulings of S0. diff --git a/design/specs/platform/vocabulary.sdp.md b/design/specs/platform/vocabulary.sdp.md index ff9a9cc..939306d 100644 --- a/design/specs/platform/vocabulary.sdp.md +++ b/design/specs/platform/vocabulary.sdp.md @@ -70,7 +70,7 @@ The two clashes are answered: a row that proves an effect happened is completion - **rollback** — Making a retired generation the one readers see again (D9). - **retired generation** — A generation readers no longer see, kept until it is purged (D9). - **purge** — Deleting a retired generation's rows (D9). -- **fence** — The number a generation's batches carry so that a stale batch writes nothing (D9). +- **fence** — The number a generation's batches carry so that a stale batch writes no read-model row and schedules nothing (D9). - **progress row** — The row that holds where a rebuild stands, read by its batches and by no command (D9). - **tenant list** — The parent's list of tenants, which a rebuild and a sweep walk (D11, D19). - **backup archive** — The exported file of a deployment's table data at one moment, which a restore imports; a snapshot is a transaction's view of the database, never the archive (D19). From 77acdf22211656ede5f1d8d1a96b881ce581506d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 3 Oct 2026 12:14:55 +0200 Subject: [PATCH 153/155] build: the pull request review's three fixes - The timing command no longer forces the owner's Linux backend binary. The harness picks the pinned release for the platform and downloads it into .cache/, or takes CONVEX_BACKEND_BINARY when the caller sets it. - Probe 7's hosted in-place example covers depot/migrations, the migrations component the depot mounts. The archive parser reads nested _components//_components// paths, the scheduler rows are read in that scope, and the binding counts scopes by the key's last slash. A throwaway run on a local native backend running the hosted copy passed with depot/migrations/migrations compared; the hosted deployment has not run it. - STATE.md takes the corpus digest the check prints, 90b4b34a0c53c54a, and says what the review found. Co-Authored-By: Claude Opus 5.5 --- design/STATE.md | 4 +++- scripts/measure-place-order.mjs | 7 ------- tests/hosted/examples.ts | 19 ++++++++++++++----- tests/hosted/hosted-examples.test.ts | 9 +++++++-- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/design/STATE.md b/design/STATE.md index 77aea81..dc4e86b 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -11,7 +11,7 @@ open questions: 69 Specs, 153 questions, 9 blocking; extensions registered: 54 stated readiness: {'defined': 227, 'idea': 1, 'scoped': 9} ledger: 278 findings, {'fixed': 159, 'open': 80, 'owner': 1, 'partially-fixed': 38} open findings by slice: {'L3': 30, 'P': 29, 'S3': 1, 'S4': 4, 'S5': 16} -corpus digest: 2851beee8cf5338b +corpus digest: 90b4b34a0c53c54a protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` @@ -28,6 +28,8 @@ advisor: convex=37, domain=45, operator=34, product=39 CI passed on the branch's pushes in round 2 (`1b92bd8`, `a5d339d`) and in 3b (`314c265`, `2283083`, `de285bb`, and its close `840b32c` in 7 m 12 s). It failed on the push of sitting two, `d2a8d1b` (run 37105650743): two native files of 166, Probe 4's nested-call example and Sc L2-5's `rebuild.online-rebuild-interrupt-resume`, whose `interruptGeneration` lost every rerun to the backfill batches on `generationProgress` (`OptimisticConcurrencyControlFailure`), so acceptance read 31 passed, 1 failed and 6 missing (Leads 43). Nothing was pushed after it until this close. CI on the close's push, `1b3ff58` (run 37112192453), failed one file of 175, Probe 4's nested file on its 1 s limit, and passed L2-5. Two fixes followed before the pull request, each read by sol: Probe 4's documents hold an ASCII string in place of bytes, so the step reaches the bytes-written limit in about 0.06 s where it took 0.44 s (`4ff759d`); and `interruptGeneration` reads and writes the generation row only, marking `interruptedFence`, while a batch that finds its fence stale records the interruption on the progress row, so the interrupt no longer conflicts with the batches on `generationProgress` (`2a44655`, `131b48a`). CI on `131b48a` (run 37113652222, 7 m 17 s) passed: `npm test` 69 files; the whole tier 175 files, 174 passed and 1 skipped, 1,201 tests passed and 5 skipped; acceptance 34 passed, 0 failed, 0 absent, 4 missing, "run passed · tree clean". The local whole-tier runs of this unit ran under load and are no proof of the whole tier; CI's runs are. +A review of the pull request at `9e58019` found three things, and the commit after it fixes them. The timing command forced the path of the owner's Linux backend binary, so a fresh clone, another checkout or a Mac failed. It now leaves the choice to the harness, which downloads the pinned release into `.cache/`. Probe 7's hosted in-place example skipped `depot/migrations`, the migrations component the depot mounts, because the archive names its tables `_components/depot/_components/migrations/` and the parser read only one level. The example now reads, compares and asserts on that scope too. A throwaway run of it on a local native backend running the hosted copy passed, with `depot/migrations/migrations` among the compared tables; the hosted deployment has not run it. And this file kept the corpus digest of `1b3ff58`, from before the two fixes and the close edited Specs. + The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Slice S0 reached it through a pull request; from S1 on the session fast-forwarded it once CI was green, on the owner's mandate of 2026-10-01, until the merge of slice S2 and the platform review unit on 2026-10-02, when the owner said: "Please merge to main and push. This is the last change we will do without formal PRs since things are starting to shape up." Round 1's four lanes then reached it through pull requests #6 to #9 (decision 32). The platform review as it was received is in commit `70c15bc`, with paths of the owner's machine in its links; its files left the tree at that unit's close, and the owner said the deleted review may stay in the history. ## Where the work stands diff --git a/scripts/measure-place-order.mjs b/scripts/measure-place-order.mjs index fd9bb17..3a1c955 100644 --- a/scripts/measure-place-order.mjs +++ b/scripts/measure-place-order.mjs @@ -1,6 +1,4 @@ import { spawnSync } from "node:child_process"; -import { homedir } from "node:os"; -import { join } from "node:path"; const environment = { ...process.env }; for (const key of [ "CONVEX_DEPLOY_KEY", @@ -12,11 +10,6 @@ environment.PLACE_ORDER_TIMINGS = "1"; environment.PLACE_ORDER_TIMING_LOAD = process.argv.includes("--quiet") ? "quiet" : "under load"; -// The pinned backend binary of the owner's machine, unless the caller exported another path. -environment.CONVEX_BACKEND_BINARY ??= join( - homedir(), - "dev-libar/libar-platform/.cache/precompiled-2026-09-28-5c7cb5b/convex-local-backend-x86_64-unknown-linux-gnu.zip/convex-local-backend", -); const child = spawnSync( process.execPath, [ diff --git a/tests/hosted/examples.ts b/tests/hosted/examples.ts index e336d85..3091125 100644 --- a/tests/hosted/examples.ts +++ b/tests/hosted/examples.ts @@ -146,25 +146,34 @@ export function ownStates( } // Probe 7's replacement in place. -// The copy mounts these components beside the parent; a component that another composition's -// deploy left unmounted is not among them. +// The copy mounts these components beside the parent, a nested one by its path; a component that +// another composition's deploy left unmounted is not among them. export const mountedComponents = [ "annex", "annexClock", "migrations", "depot", + "depot/migrations", "yard", ]; const scopeName = (scope: CopyScope) => scope ?? "parent"; // Every user table of the parent and of every mounted component, read whole, keyed by -// scope/table, from the tables a backup archive holds. +// scope/table, from the tables a backup archive holds. The archive names a nested component's table +// _components/depot/_components/migrations/
. async function storedData(backend: Backend, entries: string[]) { const data: Record = {}; for (const entry of entries) { const match = - /^(?:_components\/([^/]+)\/)?([^_/][^/]*)\/documents\.jsonl$/.exec(entry); + /^((?:_components\/[^/]+\/)*)([^_/][^/]*)\/documents\.jsonl$/.exec(entry); if (match === null) continue; - const [, component, table] = match; + const [, prefix, table] = match; + const component = + prefix === "" + ? undefined + : prefix! + .slice("_components/".length, -1) + .split("/_components/") + .join("/"); if (component !== undefined && !mountedComponents.includes(component)) continue; data[`${component ?? "parent"}/${table!}`] = await backend.admin.readTable( diff --git a/tests/hosted/hosted-examples.test.ts b/tests/hosted/hosted-examples.test.ts index cda3077..b47373f 100644 --- a/tests/hosted/hosted-examples.test.ts +++ b/tests/hosted/hosted-examples.test.ts @@ -83,13 +83,15 @@ bindExample(probe7HostedRetentionContract, (): RetentionWorld => ({}), { ownStates(world, states, completedTime), }); -// The scopes the copy mounts: the parent and every component of the fixture composition. +// The scopes the copy mounts: the parent and every component of the fixture composition, the +// migrations component the depot mounts included. const copyMounts = [ "parent", "annex", "annexClock", "migrations", "depot", + "depot/migrations", "yard", ]; type InPlace = InPlaceWorld & { @@ -112,8 +114,11 @@ bindExample(probe7HostedInPlaceImportContract, (): InPlace => ({}), { (world, { scopes, equal }) => { const o = observed(world); expect(scopes).toBe("parent and every mounted component"); + // A key is scope/table, and a nested scope holds a slash of its own. const read = new Set( - Object.keys(o.exported).map((key) => key.split("/")[0]), + Object.keys(o.exported).map((key) => + key.slice(0, key.lastIndexOf("/")), + ), ); expect(read).toEqual(new Set(copyMounts)); expect(isDeepStrictEqual(o.restored, o.exported)).toBe(equal); From bb82f33577a49c2689f32e8d89610c86c44107f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 10 Oct 2026 11:23:35 +0200 Subject: [PATCH 154/155] fix(receipts): the sweep reads the gate before it deletes The receipt sweep deleted expired receipts without reading the maintenance gate, and sweepNext reached it through sweep, while spec:application.write-pause promises that every writer in a scope reads the gate inside its own mutation and the restore door keeps every writer out. The Spec side is right (review of 2026-10-10, F2). - sweep calls assertWritable after its argument check and before its first receipt read, with the scopes "all" and the tenant's and no source scope, because it writes no source stream; a closed scope refuses the run with writePaused and it deletes nothing. sweepNext reads the gate through sweep, after its tenant-list read. - The write-pause, receipt-table and idempotency-and-receipts Specs say so where they describe the gate's callers and the sweep's reads; the gate's header comment names the sweep among the callers. - convex-test, both compositions: a closed deployment and a closed tenant refuse sweep and sweepNext with the transient writePaused and the receipts remain; the restore door refuses with "write paused for all: restore"; the same call deletes after the resume or once the variable is cleared. Removing the gate read fails all six and no other test. - Native, tests/native/gate.test.ts: the restore door refuses the sweep on a running backend and it deletes nothing, and the same call deletes the expired receipt once the variable is cleared. - STATE.md records the repair and its re-measured corpus digest. Co-Authored-By: Claude Fable 5.1 --- design/STATE.md | 4 +- design/specs/application/write-pause.sdp.md | 6 +- .../command/idempotency-and-receipts.sdp.md | 6 +- design/specs/command/receipt-table.sdp.md | 6 +- src/command/receipts.ts | 22 ++- src/gate/gate.ts | 8 +- tests/native/gate.test.ts | 36 +++++ tests/simulator/receipts-sweep.test.ts | 145 +++++++++++++++++- 8 files changed, 214 insertions(+), 19 deletions(-) diff --git a/design/STATE.md b/design/STATE.md index dc4e86b..9235a4a 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -11,7 +11,7 @@ open questions: 69 Specs, 153 questions, 9 blocking; extensions registered: 54 stated readiness: {'defined': 227, 'idea': 1, 'scoped': 9} ledger: 278 findings, {'fixed': 159, 'open': 80, 'owner': 1, 'partially-fixed': 38} open findings by slice: {'L3': 30, 'P': 29, 'S3': 1, 'S4': 4, 'S5': 16} -corpus digest: 90b4b34a0c53c54a +corpus digest: c8bdcdb0426e0bf8 protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` @@ -30,6 +30,8 @@ CI passed on the branch's pushes in round 2 (`1b92bd8`, `a5d339d`) and in 3b (`3 A review of the pull request at `9e58019` found three things, and the commit after it fixes them. The timing command forced the path of the owner's Linux backend binary, so a fresh clone, another checkout or a Mac failed. It now leaves the choice to the harness, which downloads the pinned release into `.cache/`. Probe 7's hosted in-place example skipped `depot/migrations`, the migrations component the depot mounts, because the archive names its tables `_components/depot/_components/migrations/
` and the parser read only one level. The example now reads, compares and asserts on that scope too. A throwaway run of it on a local native backend running the hosted copy passed, with `depot/migrations/migrations` among the compared tables; the hosted deployment has not run it. And this file kept the corpus digest of `1b3ff58`, from before the two fixes and the close edited Specs. +A review of 2026-10-10 across the four open pull requests found one contract break on this branch, and the commit after `77acdf2` repairs it. The receipt sweep deleted expired receipts without reading the maintenance gate, and `sweepNext` reached it, while `spec:application.write-pause` promises that every writer in a scope reads the gate inside its own mutation, names the mutations that skip the read with the sweep among none of them, and says the restore door keeps every writer out. The Spec side was right. `sweep` now reads the gate after its argument check and before its first receipt read, the restore door first, for the scopes `all` and the tenant's and no source scope, and `sweepNext` reads it through `sweep` after the tenant-list read; a closed scope refuses the run with `writePaused` and it deletes nothing. The three Specs that describe the sweep, the gate's header comment, the `convex-test` suite of the sweep on both compositions and one native test of the restore door against the sweep changed in the same commit, and the corpus digest below is re-measured. Drift repair, not a decision: no register row. + The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Slice S0 reached it through a pull request; from S1 on the session fast-forwarded it once CI was green, on the owner's mandate of 2026-10-01, until the merge of slice S2 and the platform review unit on 2026-10-02, when the owner said: "Please merge to main and push. This is the last change we will do without formal PRs since things are starting to shape up." Round 1's four lanes then reached it through pull requests #6 to #9 (decision 32). The platform review as it was received is in commit `70c15bc`, with paths of the owner's machine in its links; its files left the tree at that unit's close, and the owner said the deleted review may stay in the history. ## Where the work stands diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index 7d60c78..3478755 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -37,7 +37,7 @@ The write pause is the maintenance gate every writer in a scope reads inside its ## Contract -- There is one gate document per deployment, read through `assertWritable`, which throws, by the command pipeline at its step 7 and by the grant helpers `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` before they write, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches, its tenant fill's `fillTenantsBatch`, and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries, the rebuild's `fillTenants`, which reads the restore door alone and is refused while it is closed, and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states; every command that reaches step 7 reads the gate; a duplicate answered from its receipt does not, because its answer writes nothing (D9, D6, E-8, E-25, E-37, E-42) +- There is one gate document per deployment, read through `assertWritable`, which throws, by the command pipeline at its step 7, by the grant helpers `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` before they write, and by the receipt sweep `sweep` of `spec:command.receipt-table`, which `sweepNext` runs, after its argument check and before it reads a receipt, and by every background batch at the start of its mutation through `gateAllows`, which answers and lets the batch park, never cached across mutations; the background readers are the rebuild's backfill, verify and purge batches, its tenant fill's `fillTenantsBatch`, and the journal's baseline driver of `spec:context.journal`, which writes a source stream and reschedules itself when refused; the only mutations that skip the read are the gate's own entries, the rebuild's `fillTenants`, which reads the restore door alone and is refused while it is closed, and the restore procedure's `startRestoreRun`, its check batches, its pending-work inventory and `finishRestoreRun`, which write only the `restoreRuns` record and the generation rows the accepted branch resumes, with their progress rows, while the restore door is closed, as `spec:application.restore` states; every command that reaches step 7 reads the gate; a duplicate answered from its receipt does not, because its answer writes nothing (D9, D6, E-8, E-25, E-37, E-42) - A scope is `all`, one tenant, or one source stream type of one context; the gate holds the set of closed scopes, each with a reason, the generation it serves if any, and the operator who closed it (E-8, E-48, D9) - A writer in a closed scope refuses before any write with the transient refusal `writePaused`, whose message carries the scope and the reason; nothing is stored, no operation ID is minted, and the same request key succeeds once the scope is open (D9, D4, Law 6) - A use case is in scope when `all` is closed, when its tenant is closed, or when a stream type its declaration's `writes` names is closed (E-8, E-38, D9) @@ -65,7 +65,7 @@ The gate is one document so that a command pays one point read. Scopes are strin - indexGatesByKeyUse: the one point read every writer makes, `.withIndex("by_key", (q) => q.eq("key", "gates")).unique()`; an absent document reads as no closed scope (E-8, D9) - typeScopeKey: `type ScopeKey = "all" | string` where a tenant scope is `"tenant:" + tenantId` and a source scope is `"source:" + contextId + ":" + streamType`, built only by `scopesOfUseCase`, `startGeneration`, the rebuild's batches, the journal's baseline driver and the operator entries (E-8, E-38, D11) - fnGateAllows: `gateAllows(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[], allowGeneration?: Id<"generations">): Promise<{ allowed: true } | { allowed: false; scopeKey: ScopeKey; reason: string }>` the one gate read: it first reads `process.env.MAINTENANCE_MODE` and answers `{ allowed: false, scopeKey: "all", reason: "restore" }` without reading the document when the value is `restore`; otherwise it reads the gate document once and answers not allowed with the `scopeKey` and `reason` of the first entry of `closed`, in array order, whose `scopeKey` is one of `scopes` and which is not exempt, and allowed when there is none; an entry is exempt only when `allowGeneration` is given, the entry has a `generationId`, and the two are equal; it writes nothing (E-8, E-42, D9, F9) -- fnAssertWritable: `assertWritable(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[]): Promise` calling `gateAllows` with no generation and, when the answer is not allowed, throwing through the boundary's `refuseTransient({ code: "writePaused", message: "write paused for " + scopeKey + ": " + reason })`; the command pipeline's step 7 calls it before it mints the operation, and `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` call it with `scopesOfUseCase(grant.tenantId, [])` before their first read; nothing else does (E-8, D9, E-35, E-37) +- fnAssertWritable: `assertWritable(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[]): Promise` calling `gateAllows` with no generation and, when the answer is not allowed, throwing through the boundary's `refuseTransient({ code: "writePaused", message: "write paused for " + scopeKey + ": " + reason })`; the command pipeline's step 7 calls it before it mints the operation, and `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` call it with `scopesOfUseCase(grant.tenantId, [])` before their first read, and the receipt sweep `sweep` of `spec:command.receipt-table` calls it with `["all", tenantScope(tenantId)]`, no source scope because it writes no source stream, after its argument check and before its first receipt read, so `sweepNext` reads it through `sweep` after its tenant-list read; nothing else does (E-8, D9, E-35, E-37) - errorCodeWritePaused: `writePaused`, the transient code the outcome boundary reserves; the refusal's data is exactly `{ kind: "transient", code: "writePaused", message }` with no `retryAfterMs`, where `message` is `"write paused for all: restore"` under the restore door and `"write paused for tenant:" + tenantId + ": " + reason` for a closed tenant; a transient refusal, never a stored rejection, so no receipt, event, read-model row or audit record exists for the refused command (D4, Law 6, E-35) - restoreDoor: `gateAllows` runs in the parent and reads `process.env.MAINTENANCE_MODE` before the gate document; only the exact value `restore` closes the door, so the restore procedure keeps writers out even though the backup archive replaced the gate, and every rebuild batch parks until the accepted branch resumes it; the restore procedure's own six mutations call neither `gateAllows` nor `assertWritable`; no component reads the variable (E-42, D19, F11) - fnScopesOfUseCase: `scopesOfUseCase(tenantId: string, writes: readonly SourceRef[]): ScopeKey[]` returning `"all"`, the tenant scope and one source scope per `SourceRef` of the declaration's `writes`, in that order, so a command is held by a closed deployment, by its closed tenant and by a paused rebuild of any view whose sources it writes (E-8, E-38, D9) @@ -83,7 +83,7 @@ The gate is one document so that a command pays one point read. Scopes are strin ## Verification — reviewed -- A reviewer confirms that `assertWritable` is called by the command pipeline's step 7 for every command that reaches it and by the two grant helpers before they write, that `gateAllows` is called by every background batch before its first write and that the batch parks rather than throws on a refusal, that the two are one read because `assertWritable` is `gateAllows` plus the throw, that the gate's entries and the restore procedure's six mutations are the only exemptions, and that no function caches the gate across mutations. +- A reviewer confirms that `assertWritable` is called by the command pipeline's step 7 for every command that reaches it, by the two grant helpers before they write and by the receipt sweep before it reads a receipt, that `gateAllows` is called by every background batch before its first write and that the batch parks rather than throws on a refusal, that the two are one read because `assertWritable` is `gateAllows` plus the throw, that the gate's entries and the restore procedure's six mutations are the only exemptions, and that no function caches the gate across mutations. - A reviewer confirms that `gateAllows` reads the environment before the document, that an entry with no generation is never exempt, and that a gate read with no generation passes no closed entry. - A reviewer confirms that no function other than `startGeneration` closes a source scope, so every closed entry that names a generation names a row that exists, and that the journal's baseline driver reads the gate with the source scope of the stream type it migrates before every batch. - A reviewer confirms that `closeScope` and `resumeScope` are the only writers of the gate document, that each inserts its audit record in the same mutation with no catch around it, and that a test whose audit table refuses the record finds the gate unchanged. diff --git a/design/specs/command/idempotency-and-receipts.sdp.md b/design/specs/command/idempotency-and-receipts.sdp.md index 9cfe2db..c970e03 100644 --- a/design/specs/command/idempotency-and-receipts.sdp.md +++ b/design/specs/command/idempotency-and-receipts.sdp.md @@ -80,10 +80,10 @@ This Spec owns the rules: where receipts are required, how the key and the finge The receipt read and insert are helpers inside the pipeline's mutation, not registered functions. The key is four indexed fields rather than a concatenated string, so no delimiter can collide and the index answers the lookup in one range. The fingerprint is a digest so that a receipt never holds business input or personal data; a conflict answer names nothing of the original except the one fact its code cannot hide, that the receipt's contract version is the declaration's, because `unsupportedContractVersion` answers otherwise, and the caller that sent the original finds it by sending the same key with the same input, which is answered as a duplicate. -Expiry is lazy on the core path and swept in the background. A lookup that finds an expired reversible receipt deletes it and proceeds as new intent; a lookup that finds an expired irreversible receipt compacts it to a tombstone and answers as a duplicate. The sweep is one bounded batch per tenant that removes expired reversible receipts and compacts expired irreversible ones; it is an operations duty and never decides an outcome. +Expiry is lazy on the core path and swept in the background. A lookup that finds an expired reversible receipt deletes it and proceeds as new intent; a lookup that finds an expired irreversible receipt compacts it to a tombstone and answers as a duplicate. The sweep is one bounded batch per tenant that removes expired reversible receipts and compacts expired irreversible ones; it is an operations duty and never decides an outcome. Like every writer, it reads the maintenance gate of `spec:application.write-pause` before it reads a receipt, so a pause holds it. - transactionBoundary: read-check-insert inside the pipeline's one top-level mutation; the sweep is a separate bounded mutation per tenant (D6, D19) -- convexSurface: helpers `lookupReceipt`, `classifyReceipt`, `insertReceipt`, `fingerprintOf`; two `internalMutation`s per deployment, `receipts.sweep`, one bounded run for one tenant, and `receipts.sweepNext`, one such run for the next tenant of the tenant list after the caller's cursor, as `fnSweep` and `fnSweepNext` of `spec:command.receipt-table` pin them; operations calls them, nothing schedules them, and neither is on the command path (D6, E-34) +- convexSurface: helpers `lookupReceipt`, `classifyReceipt`, `insertReceipt`, `fingerprintOf`; two `internalMutation`s per deployment, `receipts.sweep`, one bounded run for one tenant, and `receipts.sweepNext`, one such run for the next tenant of the tenant list after the caller's cursor, as `fnSweep` and `fnSweepNext` of `spec:command.receipt-table` pin them; each reads the maintenance gate of `spec:application.write-pause` after its argument check and before any receipt read, so a closed deployment, a closed tenant or the restore door refuses the run with `writePaused` and nothing is deleted; operations calls them, nothing schedules them, and neither is on the command path (D6, E-34) - typeReceiptKey: `type ReceiptKey = { tenantId: string; namespace: CallerNamespace; commandType: string; requestKey: string }` (D6, Law 4) - typeReceiptClass: `type ReceiptClass = { class: "new" } | { class: "duplicate"; receipt: Doc<"receipts"> } | { class: "tombstone"; receipt: Doc<"receipts"> } | { class: "conflict" } | { class: "unsupportedVersion" }`, where the conflict and the unsupported-version members carry no fields of the row, so no caller of `classifyReceipt` holds a field of the stored receipt to relay; the conflict and unsupported-version answers still disclose the one fact the two codes cannot hide, whether the receipt's contract version is the declaration's (D6, Law 5, E-5) - fnFingerprintOf: `fingerprintOf(input: unknown, contractVersion: number): Promise` returns the lowercase hex SHA-256 of `canonicalJson(input) + "\n" + contractVersion` (D6, E-36) @@ -99,7 +99,7 @@ Expiry is lazy on the core path and swept in the background. A lookup that finds - retentionDefault: `retention: { window: 7 * 24 * 60 * 60 * 1000, afterExpiry: "delete" | "tombstone" }` per declaration; `"tombstone"` is required when `irreversible` is true and the command has no domain uniqueness rule (D6, E-34) - limitIdLength: [extension] `limitIdLength` is 256 bytes of UTF-8, measured with `utf8Length` of `spec:command.command-pipeline`, for the tenant ID and the request key, a longer one is rejection `invalidInput` at step 1, and for the declaration's name, a longer one is a plain error; the count is of bytes, never of characters, so one number sizes the receipt row and the event envelope (E-34) - limitAffectedRefs: at most 1,000 affected refs and 1,000 stream versions per receipt, checked by `insertReceipt`; the row-size arithmetic against the 1 MiB document limit is `limitRowSize` in `spec:command.receipt-table`, and this Spec states no figure of its own (F13, E-34) -- limitSweepBatch: `receipts.sweep` handles at most 1,000 receipts per run for one tenant, reading `by_tenant_expiry` up to `now`, so one run stays far under the 16,000 documents written per transaction; its read is bounded by bytes as well, through `boundedPage` with `limitListBytes` of `spec:context.queries`, because 1,000 rows near the row bound of `limitRowSize` would pass the 16 MiB read limit, so a run ends at 1,000 rows or at 8 MiB read, whichever comes first, and answers `more`; the limit is a ceiling, not a target (D19, F13, E-34) +- limitSweepBatch: `receipts.sweep` handles at most 1,000 receipts per run for one tenant, reading `by_tenant_expiry` up to `now` once the gate read has passed, so one run stays far under the 16,000 documents written per transaction; its read is bounded by bytes as well, through `boundedPage` with `limitListBytes` of `spec:context.queries`, because 1,000 rows near the row bound of `limitRowSize` would pass the 16 MiB read limit, so a run ends at 1,000 rows or at 8 MiB read, whichever comes first, and answers `more`; the limit is a ceiling, not a target (D19, F13, E-34) - serializationArgument: both concurrent requests read the `by_key` range and find nothing; the first to commit inserts into that range; the second's commit is refused because its read set changed, a read set that holds the receipt range and the context's rows, so the conflict the engine reports may name either, the engine retries it, and the retry reads the receipt; the engine retry is invisible to the caller and is not a business attempt (F1, D15) - exampleActorKinds: the examples of this Spec's space run a caller in namespace `public` as a `human` actor, `worker` and `service` as a `service` actor and `agent` as an `agent` actor; the internal entry takes the actor and the namespace as two arguments, so the namespace alone fixes no actor kind (Sc L1-3, D11, E-6) diff --git a/design/specs/command/receipt-table.sdp.md b/design/specs/command/receipt-table.sdp.md index 34591b0..4f65bd1 100644 --- a/design/specs/command/receipt-table.sdp.md +++ b/design/specs/command/receipt-table.sdp.md @@ -37,7 +37,7 @@ The `receipts` table lives in the parent, as the doc's ownership table assigns i - A row holds `recordedAt` and `expiresAt`; a lookup treats a row at or past `expiresAt` as expired (D6, E-4) - A tombstone is a row with `tombstone` true, `affected` and `versions` empty, and `expiresAt` at the tombstone horizon (D6, E-4) - Rows are found for diagnosis by operation ID through `by_operation`, because stored receipts are the authority for what committed (D19) -- The per-tenant sweep reads `by_tenant_expiry` up to the current time and handles at most 1,000 rows per run (D19, F13, E-4) +- The per-tenant sweep is a writer of `spec:application.write-pause`: it reads the maintenance gate before it reads a receipt, so a closed deployment, a closed tenant or the restore door refuses it with `writePaused` and it deletes nothing; otherwise it reads `by_tenant_expiry` up to the current time and handles at most 1,000 rows per run (D19, F13, E-4) - [extension] The default retry window is 7 days and the default tombstone horizon is never, both per declaration (E-4) ## Design @@ -56,8 +56,8 @@ The `receipts` table lives in the parent, as the doc's ownership table assigns i - expiryDefaults: `expiresAt = recordedAt + retention.window` with a default window of 7 days; `tombstoneHorizon` defaults to `Number.MAX_SAFE_INTEGER`, meaning never, and a declaration may set a finite horizon (D6, E-4) - limitRowSize: [extension] at most 1,000 affected refs and 1,000 versions, the counts `insertReceipt` checks, keep a row under 400 KiB, below the 1 MiB document limit with room for the key fields, only while each entry measures under 200 bytes; the tenant ID, the request key and the command type are each at most `limitIdLength` bytes of UTF-8, checked at the pipeline's step 1; a stream ID inside an entry is at most 256 bytes, checked at the persistence adapter's step 1, and no check holds an entry under 200 bytes, so a row above the document limit fails at its insert as a technical failure; this is the one figure for receipt size in the corpus, and `spec:command.idempotency-and-receipts` cites it rather than restating it (F13, E-4) - limitIndexes: three indexes of at most four fields, within the 32 indexes and 16 fields per index the platform allows (F13) -- fnSweep: `export const sweep = internalMutation({ args: { tenantId: v.string(), now: v.number(), limit: v.number() }, returns: v.object({ deleted: v.number(), compacted: v.number(), more: v.boolean() }), handler })` with `limit` capped at 1,000; it refuses with a plain error a `now` that is not finite or is later than the mutation's `Date.now()` and a `limit` that is not a positive safe integer, and deletes nothing then; it reads expired rows through `boundedPage` of `spec:context.queries` with the byte bound `limitListBytes`, so a run ends at `min(limit, 1000)` rows or at that byte bound, whichever comes first, deletes each row it read and answers `more` when an expired row remains (D19, E-4, F13) -- fnSweepNext: `export const sweepNext = internalMutation({ args: { after: v.union(v.string(), v.null()), now: v.number(), limit: v.number() }, returns: v.object({ deleted: v.number(), compacted: v.number(), more: v.boolean(), tenantId: v.union(v.string(), v.null()), after: v.union(v.string(), v.null()) }), handler })` checks `now` and `limit` as `sweep` does before anything else, takes the first tenant after `after` from the tenant list through `nextTenant` and runs one `sweep` for it, one tenant per run; it answers that tenant with the sweep's counts and the cursor for the next run, which stays `after` while the tenant has `more` and becomes the tenant otherwise, and a `tenantId` of `null` with zero counts, `more` false and `after` null when no tenant follows, which ends the pass; the caller holds the cursor between runs, and nothing schedules a run (D19, E-4, Law 11) +- fnSweep: `export const sweep = internalMutation({ args: { tenantId: v.string(), now: v.number(), limit: v.number() }, returns: v.object({ deleted: v.number(), compacted: v.number(), more: v.boolean() }), handler })` with `limit` capped at 1,000; it refuses with a plain error a `now` that is not finite or is later than the mutation's `Date.now()` and a `limit` that is not a positive safe integer, and deletes nothing then; after that check and before any receipt read it calls `assertWritable` of `spec:application.write-pause` with the scopes `all` and `tenantScope(tenantId)`, the restore door read first, and a closed scope refuses the run with the transient `writePaused` and deletes nothing; it reads expired rows through `boundedPage` of `spec:context.queries` with the byte bound `limitListBytes`, so a run ends at `min(limit, 1000)` rows or at that byte bound, whichever comes first, deletes each row it read and answers `more` when an expired row remains (D19, E-4, F13) +- fnSweepNext: `export const sweepNext = internalMutation({ args: { after: v.union(v.string(), v.null()), now: v.number(), limit: v.number() }, returns: v.object({ deleted: v.number(), compacted: v.number(), more: v.boolean(), tenantId: v.union(v.string(), v.null()), after: v.union(v.string(), v.null()) }), handler })` checks `now` and `limit` as `sweep` does before anything else, takes the first tenant after `after` from the tenant list through `nextTenant` and runs one `sweep` for it, one tenant per run, which reads the gate for that tenant after the tenant-list read and before any receipt read, so a closed scope refuses the run with `writePaused` and nothing is deleted; it answers that tenant with the sweep's counts and the cursor for the next run, which stays `after` while the tenant has `more` and becomes the tenant otherwise, and a `tenantId` of `null` with zero counts, `more` false and `after` null when no tenant follows, which ends the pass; the caller holds the cursor between runs, and nothing schedules a run (D19, E-4, Law 11) ## Verification — reviewed diff --git a/src/command/receipts.ts b/src/command/receipts.ts index 654f093..65997f9 100644 --- a/src/command/receipts.ts +++ b/src/command/receipts.ts @@ -2,9 +2,11 @@ // classification and the insert, helpers inside the pipeline's mutation; and the sweep and its loop // over tenants, handlers that each composition registers as internal mutations for operations. import { paginator } from "convex-helpers/server/pagination"; -import { defineSchema } from "convex/server"; +import { defineSchema, type GenericDatabaseReader } from "convex/server"; import { convexToJson, v, type JSONValue, type Value } from "convex/values"; import { boundedPage, limitListBytes } from "../context/queries.js"; +import { assertWritable, tenantScope } from "../gate/gate.js"; +import type { GateDataModel } from "../gate/tables.js"; import type { AffectedRef, StreamVersion } from "../kernel/index.js"; import type { CallerNamespace } from "./actor-and-scope.js"; import { nextTenant } from "./authority.js"; @@ -149,14 +151,23 @@ function checkSweepArgs(now: number, limit: number) { if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("Receipt sweep limit must be a positive safe integer"); } -// One run reads expired rows until it holds min(limit, 1000) or has read limitSweepBytes, whichever -// comes first, and deletes each row it read, an expired tombstone included. Compaction waits for -// tombstones, so compacted is 0. +// The sweep is a writer of spec:application.write-pause: after the argument check and before any +// receipt read it reads the gate, the restore door first, for "all" and the tenant's scope and no +// source scope, because it writes no source stream; a closed scope refuses the run with writePaused +// and it deletes nothing. One run then reads expired rows until it holds min(limit, 1000) or has +// read limitSweepBytes, whichever comes first, and deletes each row it read, an expired tombstone +// included. Compaction waits for tombstones, so compacted is 0. export async function sweep( ctx: MutationCtx, { tenantId, now, limit }: { tenantId: string; now: number; limit: number }, ): Promise { checkSweepArgs(now, limit); + // The library types ctx with the command tables; every composition that registers the sweep holds + // the gate's table too, and at run time ctx is that composition's own. + await assertWritable( + ctx as unknown as { db: GenericDatabaseReader }, + ["all", tenantScope(tenantId)], + ); const expired = () => paginator(ctx.db, commandSchema) .query("receipts") @@ -177,7 +188,8 @@ export async function sweep( }; } // The operations caller carries after between runs. A tenant with more stays on the same cursor; -// a null tenant ends the pass. Nothing schedules another run. +// a null tenant ends the pass. Nothing schedules another run. The gate is read through sweep, after +// the tenant-list read, which is a read. export const sweepNextArgs = { after: v.union(v.string(), v.null()), now: v.number(), diff --git a/src/gate/gate.ts b/src/gate/gate.ts index 4eb1120..c841da6 100644 --- a/src/gate/gate.ts +++ b/src/gate/gate.ts @@ -1,8 +1,10 @@ // The writer check and the two gate writers of spec:application.write-pause. gateAllows is the one gate // read: the restore door through the environment first, then the gate document. assertWritable is -// gateAllows plus the throw, called by the command pipeline's step 7 and by nothing else. closeScope and -// resumeScope are the only writers of the gate document, and each writes its operator audit record in -// the caller's mutation with no catch around it, so a failed insert changes nothing. +// gateAllows plus the throw, called by the command pipeline's step 7, by the grant helpers insertGrant +// and revokeGrant before their first read, by the receipt sweep before its first receipt read, and by +// nothing else. closeScope and resumeScope are the only writers of the gate document, and each writes +// its operator audit record in the caller's mutation with no catch around it, so a failed insert +// changes nothing. import type { GenericDatabaseReader, GenericDatabaseWriter, diff --git a/tests/native/gate.test.ts b/tests/native/gate.test.ts index e2fc51f..547f260 100644 --- a/tests/native/gate.test.ts +++ b/tests/native/gate.test.ts @@ -330,6 +330,42 @@ test("native: two independent writers race a close and no successful command com // cannot witness the concurrency obligation. expect(overlapping + reruns).toBeGreaterThan(0); }); +// spec:application.write-pause restoreDoor, Contract "The restore procedure cannot use the gate document"; spec:command.receipt-table fnSweep: the receipt sweep is a writer, and the restore door keeps it out. +test("native: the restore door refuses the receipt sweep and it deletes nothing, and the same call deletes the expired receipt once the variable is cleared", async () => { + const backend = await fixtureBackend(); + await backend.admin.writeTable("receipts", { + insert: { + tenantId: "t", + namespace: "worker", + commandType: "SweepTest", + requestKey: "expired", + fingerprint: "fingerprint", + contractVersion: 1, + outcome: "applied", + operationId: "expired", + affected: [], + versions: [], + actorId: "worker", + recordedAt: 0, + expiresAt: 1, + tombstone: false, + }, + }); + const before = await stored(backend); + expect(before.receipts).toHaveLength(1); + const run = () => + backend.admin.run(getFunctionName(internal.receipts.sweep), { + tenantId: "t", + now: Date.now() - 60_000, + limit: 1000, + }); + await backend.admin.setEnvironment({ MAINTENANCE_MODE: "restore" }); + await expectPaused(run(), "all", "restore"); + expect(await stored(backend)).toEqual(before); + await backend.admin.setEnvironment({ MAINTENANCE_MODE: "" }); + expect(await run()).toEqual({ deleted: 1, compacted: 0, more: false }); + expect(await backend.admin.readTable("receipts")).toEqual([]); +}); function overlap(a: CompletionRecord, b: CompletionRecord) { return ( a.timestamp - a.executionTime < b.timestamp && diff --git a/tests/simulator/receipts-sweep.test.ts b/tests/simulator/receipts-sweep.test.ts index ac291ed..0aac5b9 100644 --- a/tests/simulator/receipts-sweep.test.ts +++ b/tests/simulator/receipts-sweep.test.ts @@ -2,10 +2,11 @@ import { convexTest, type TestConvex } from "convex-test"; import type { SchemaDefinition } from "convex/server"; import { getDocumentSize } from "convex/values"; import { commandTables } from "../../src/command/tables.js"; -import { expect, test } from "vitest"; +import { afterEach, expect, test, vi } from "vitest"; import schema from "../../fixture/convex/schema.js"; import { internal as fixtureInternal } from "../../fixture/convex/_generated/api.js"; import { internal as productionInternal } from "../../example/convex/_generated/api.js"; +import { refusal } from "./gate-support.js"; import { productionTest } from "./production.js"; const compositions: { @@ -22,6 +23,7 @@ const compositions: { { name: "production", app: productionTest, internal: productionInternal }, ]; const now = 1000; +afterEach(() => vi.unstubAllEnvs()); function receipt(tenantId: string, requestKey: string, expiresAt = now) { return { tenantId, @@ -326,4 +328,145 @@ for (const composition of compositions) { await t.run((ctx) => ctx.db.query("receipts").collect()), ).toHaveLength(1); }); + // spec:application.write-pause fnAssertWritable, errorCodeWritePaused; spec:command.receipt-table + // fnSweep, fnSweepNext: the sweep reads the gate after its argument check and before any receipt read. + test(`convex-test: ${composition.name} a closed deployment refuses sweep and sweepNext with writePaused and the expired receipts remain until the resume`, async () => { + const t = composition.app(); + const stored = await t.run(async (ctx) => { + await ctx.db.insert("tenants", { tenantId: "a", createdAt: 0 }); + for (let i = 0; i < 3; i++) + await ctx.db.insert("receipts", receipt("a", `expired-${i}`)); + return ctx.db.query("receipts").collect(); + }); + await t.mutation(composition.internal.gate.closeGate, { + scopeKey: "all", + reason: "move", + operator: "ops", + }); + const run = () => + t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 1000, + }); + // The argument check comes first: a bad limit is the plain error, not the refusal. + await expect( + t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 0, + }), + ).rejects.toThrow("positive safe integer"); + await refusal(run(), "all", "move"); + await refusal( + t.mutation(composition.internal.receipts.sweepNext, { + after: null, + now, + limit: 1000, + }), + "all", + "move", + ); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual( + stored, + ); + await t.mutation(composition.internal.gate.resumeGate, { + scopeKey: "all", + operator: "ops", + }); + expect(await run()).toEqual({ deleted: 3, compacted: 0, more: false }); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual( + [], + ); + }); + // spec:application.write-pause Contract "A use case is in scope when all is closed, when its tenant is closed"; spec:command.receipt-table fnSweep, fnSweepNext. + test(`convex-test: ${composition.name} a closed tenant refuses its sweep and the loop sweeps another tenant`, async () => { + const t = composition.app(); + const kept = await t.run(async (ctx) => { + for (const tenantId of ["a", "b"]) { + await ctx.db.insert("tenants", { tenantId, createdAt: 0 }); + await ctx.db.insert( + "receipts", + receipt(tenantId, `${tenantId}-expired`), + ); + } + return ctx.db + .query("receipts") + .withIndex("by_tenant_expiry", (q) => q.eq("tenantId", "a")) + .collect(); + }); + await t.mutation(composition.internal.gate.closeGate, { + scopeKey: "tenant:a", + reason: "move", + operator: "ops", + }); + await refusal( + t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 1000, + }), + "tenant:a", + "move", + ); + await refusal( + t.mutation(composition.internal.receipts.sweepNext, { + after: null, + now, + limit: 1000, + }), + "tenant:a", + "move", + ); + expect( + await t.mutation(composition.internal.receipts.sweepNext, { + after: "a", + now, + limit: 1000, + }), + ).toEqual({ + tenantId: "b", + after: "b", + deleted: 1, + compacted: 0, + more: false, + }); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual( + kept, + ); + }); + // spec:application.write-pause restoreDoor, Contract "The restore door is read before the gate document"; spec:command.receipt-table fnSweep. + test(`convex-test: ${composition.name} the restore door refuses the sweep and deletes nothing, and the same call deletes once the variable is cleared`, async () => { + const t = composition.app(); + const stored = await t.run(async (ctx) => { + await ctx.db.insert("tenants", { tenantId: "a", createdAt: 0 }); + await ctx.db.insert("receipts", receipt("a", "expired")); + return ctx.db.query("receipts").collect(); + }); + vi.stubEnv("MAINTENANCE_MODE", "restore"); + const run = () => + t.mutation(composition.internal.receipts.sweep, { + tenantId: "a", + now, + limit: 1000, + }); + await refusal(run(), "all", "restore"); + await refusal( + t.mutation(composition.internal.receipts.sweepNext, { + after: null, + now, + limit: 1000, + }), + "all", + "restore", + ); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual( + stored, + ); + vi.stubEnv("MAINTENANCE_MODE", undefined); + expect(await run()).toEqual({ deleted: 1, compacted: 0, more: false }); + expect(await t.run((ctx) => ctx.db.query("receipts").collect())).toEqual( + [], + ); + }); } From 44a89c129472f2c3f7fb00f87ec1aff061955833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Darko=20Miji=C4=87?= Date: Sat, 10 Oct 2026 11:53:18 +0200 Subject: [PATCH 155/155] design(write-pause): the sweep builds a scope too A gpt-6.1-sol read of the review repair (job-mv27ccog-b3a71a66) found the typeScopeKey entry of spec:application.write-pause still saying a scope is built only by scopesOfUseCase, startGeneration, the rebuild's batches, the journal's baseline driver and the operator entries, while the repaired sweep builds its tenant scope through tenantScope. The entry now names the receipt sweep among the builders. STATE.md records the read and its re-measured corpus digest. Code and tests unchanged. Co-Authored-By: Claude Fable 5.1 --- design/STATE.md | 4 ++-- design/specs/application/write-pause.sdp.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/design/STATE.md b/design/STATE.md index 9235a4a..02dc6ea 100644 --- a/design/STATE.md +++ b/design/STATE.md @@ -11,7 +11,7 @@ open questions: 69 Specs, 153 questions, 9 blocking; extensions registered: 54 stated readiness: {'defined': 227, 'idea': 1, 'scoped': 9} ledger: 278 findings, {'fixed': 159, 'open': 80, 'owner': 1, 'partially-fixed': 38} open findings by slice: {'L3': 30, 'P': 29, 'S3': 1, 'S4': 4, 'S5': 16} -corpus digest: c8bdcdb0426e0bf8 +corpus digest: d27b24dde358fe7e protocol: file:vendor/libar-dev-software-delivery-protocol-0.0.0-5993da7.tgz ``` @@ -30,7 +30,7 @@ CI passed on the branch's pushes in round 2 (`1b92bd8`, `a5d339d`) and in 3b (`3 A review of the pull request at `9e58019` found three things, and the commit after it fixes them. The timing command forced the path of the owner's Linux backend binary, so a fresh clone, another checkout or a Mac failed. It now leaves the choice to the harness, which downloads the pinned release into `.cache/`. Probe 7's hosted in-place example skipped `depot/migrations`, the migrations component the depot mounts, because the archive names its tables `_components/depot/_components/migrations/
` and the parser read only one level. The example now reads, compares and asserts on that scope too. A throwaway run of it on a local native backend running the hosted copy passed, with `depot/migrations/migrations` among the compared tables; the hosted deployment has not run it. And this file kept the corpus digest of `1b3ff58`, from before the two fixes and the close edited Specs. -A review of 2026-10-10 across the four open pull requests found one contract break on this branch, and the commit after `77acdf2` repairs it. The receipt sweep deleted expired receipts without reading the maintenance gate, and `sweepNext` reached it, while `spec:application.write-pause` promises that every writer in a scope reads the gate inside its own mutation, names the mutations that skip the read with the sweep among none of them, and says the restore door keeps every writer out. The Spec side was right. `sweep` now reads the gate after its argument check and before its first receipt read, the restore door first, for the scopes `all` and the tenant's and no source scope, and `sweepNext` reads it through `sweep` after the tenant-list read; a closed scope refuses the run with `writePaused` and it deletes nothing. The three Specs that describe the sweep, the gate's header comment, the `convex-test` suite of the sweep on both compositions and one native test of the restore door against the sweep changed in the same commit, and the corpus digest below is re-measured. Drift repair, not a decision: no register row. +A review of 2026-10-10 across the four open pull requests found one contract break on this branch, and the commit after `77acdf2` repairs it. The receipt sweep deleted expired receipts without reading the maintenance gate, and `sweepNext` reached it, while `spec:application.write-pause` promises that every writer in a scope reads the gate inside its own mutation, names the mutations that skip the read with the sweep among none of them, and says the restore door keeps every writer out. The Spec side was right. `sweep` now reads the gate after its argument check and before its first receipt read, the restore door first, for the scopes `all` and the tenant's and no source scope, and `sweepNext` reads it through `sweep` after the tenant-list read; a closed scope refuses the run with `writePaused` and it deletes nothing. The three Specs that describe the sweep, the gate's header comment, the `convex-test` suite of the sweep on both compositions and one native test of the restore door against the sweep changed in the same commit, and the corpus digest below is re-measured. Drift repair, not a decision: no register row. A `gpt-6.1-sol` read of the repair (`job-mv27ccog-b3a71a66`) found the `typeScopeKey` entry of the write pause still naming the builders of a scope without the sweep, which builds its tenant scope through `tenantScope`; the commit after `bb82f33` adds it, and the digest below is re-measured again. The default branch is `main`, published at `github.com/libar-dev/libar-platform`. Slice S0 reached it through a pull request; from S1 on the session fast-forwarded it once CI was green, on the owner's mandate of 2026-10-01, until the merge of slice S2 and the platform review unit on 2026-10-02, when the owner said: "Please merge to main and push. This is the last change we will do without formal PRs since things are starting to shape up." Round 1's four lanes then reached it through pull requests #6 to #9 (decision 32). The platform review as it was received is in commit `70c15bc`, with paths of the owner's machine in its links; its files left the tree at that unit's close, and the owner said the deleted review may stay in the history. diff --git a/design/specs/application/write-pause.sdp.md b/design/specs/application/write-pause.sdp.md index 3478755..3e2f81a 100644 --- a/design/specs/application/write-pause.sdp.md +++ b/design/specs/application/write-pause.sdp.md @@ -63,7 +63,7 @@ The gate is one document so that a command pays one point read. Scopes are strin - validatorClosedEntry: `const closedEntryValidator = v.object({ scopeKey: v.string(), reason: v.string(), generationId: v.optional(v.id("generations")), changedAt: v.number(), changedBy: v.string() })` where `changedBy` is the stated operator and `changedAt` the closing mutation's `Date.now()` (E-8, E-48) - tableMaintenanceGates: `maintenanceGates: defineTable({ key: v.literal("gates"), closed: v.array(closedEntryValidator), updatedAt: v.number() }).index("by_key", ["key"])` holding at most one document, whose `closed` lists the entries in the order they were closed (E-8, D9) - indexGatesByKeyUse: the one point read every writer makes, `.withIndex("by_key", (q) => q.eq("key", "gates")).unique()`; an absent document reads as no closed scope (E-8, D9) -- typeScopeKey: `type ScopeKey = "all" | string` where a tenant scope is `"tenant:" + tenantId` and a source scope is `"source:" + contextId + ":" + streamType`, built only by `scopesOfUseCase`, `startGeneration`, the rebuild's batches, the journal's baseline driver and the operator entries (E-8, E-38, D11) +- typeScopeKey: `type ScopeKey = "all" | string` where a tenant scope is `"tenant:" + tenantId` and a source scope is `"source:" + contextId + ":" + streamType`, built only by `scopesOfUseCase`, `startGeneration`, the rebuild's batches, the journal's baseline driver, the receipt sweep `sweep` of `spec:command.receipt-table`, through `tenantScope`, and the operator entries (E-8, E-38, D11) - fnGateAllows: `gateAllows(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[], allowGeneration?: Id<"generations">): Promise<{ allowed: true } | { allowed: false; scopeKey: ScopeKey; reason: string }>` the one gate read: it first reads `process.env.MAINTENANCE_MODE` and answers `{ allowed: false, scopeKey: "all", reason: "restore" }` without reading the document when the value is `restore`; otherwise it reads the gate document once and answers not allowed with the `scopeKey` and `reason` of the first entry of `closed`, in array order, whose `scopeKey` is one of `scopes` and which is not exempt, and allowed when there is none; an entry is exempt only when `allowGeneration` is given, the entry has a `generationId`, and the two are equal; it writes nothing (E-8, E-42, D9, F9) - fnAssertWritable: `assertWritable(ctx: { db: GenericDatabaseReader }, scopes: readonly ScopeKey[]): Promise` calling `gateAllows` with no generation and, when the answer is not allowed, throwing through the boundary's `refuseTransient({ code: "writePaused", message: "write paused for " + scopeKey + ": " + reason })`; the command pipeline's step 7 calls it before it mints the operation, and `insertGrant` and `revokeGrant` of `spec:command.actor-and-scope` call it with `scopesOfUseCase(grant.tenantId, [])` before their first read, and the receipt sweep `sweep` of `spec:command.receipt-table` calls it with `["all", tenantScope(tenantId)]`, no source scope because it writes no source stream, after its argument check and before its first receipt read, so `sweepNext` reads it through `sweep` after its tenant-list read; nothing else does (E-8, D9, E-35, E-37) - errorCodeWritePaused: `writePaused`, the transient code the outcome boundary reserves; the refusal's data is exactly `{ kind: "transient", code: "writePaused", message }` with no `retryAfterMs`, where `message` is `"write paused for all: restore"` under the restore door and `"write paused for tenant:" + tenantId + ": " + reason` for a closed tenant; a transient refusal, never a stored rejection, so no receipt, event, read-model row or audit record exists for the refused command (D4, Law 6, E-35)