diff --git a/.changeset/29918-lifecycle-sweep-watermark.md b/.changeset/29918-lifecycle-sweep-watermark.md new file mode 100644 index 00000000000..6f74bb891a7 --- /dev/null +++ b/.changeset/29918-lifecycle-sweep-watermark.md @@ -0,0 +1,17 @@ +--- +'@objectstack/spec': minor +'@objectstack/objectql': minor +'@objectstack/platform-objects': minor +'@objectstack/cli': minor +'@objectstack/plugin-audit': minor +--- + +Clause-②: yes (widening) — `PLATFORM_OBJECTS_BY_PACKAGE['platform-objects']` (and so `PLATFORM_PROVIDED_OBJECT_NAMES` / `isPlatformProvidedObjectName`) lists one more real platform object, `sys_lifecycle_state`. Its readers resolve references against the roster and none refuses an author declaration, so a stack that references the object now resolves it instead of drawing the unknown-platform-name advisory; nothing that was accepted is refused. +Metadata-protocol: added +Skills-docs: none — `skills/**` and `content/docs/**` enumerate no platform object roster and teach nothing about the lifecycle sweep's schedule; the object is internal clock state with no data API, so there is nothing for an author to write against it. + +The lifecycle sweep remembers when it last completed, so a kernel that reboots onto an existing datastore no longer re-sweeps within a minute of every boot. + +`LifecycleService` now keeps a watermark in a new engine-owned, tenant-less object, `sys_lifecycle_state` (declared and registered by `@objectstack/platform-objects`, one row per datastore, id `lifecycle`). At the initial delay the scheduler reads that one row: when the last completed sweep is younger than the sweep interval, the first sweep waits for the interval and the read is the only statement issued. An older or missing watermark, a kernel without the object, or a read that fails (logged at `warn`) sweeps at the initial delay as before. Every sweep that completes with no policy error and a finished reference audit advances the row; a failed or partial sweep leaves it alone. A datastore the process created from empty still defers its first sweep one interval, and now seeds the row. A direct `sweep()` call still always sweeps. + +The row is internal state, not a setting: no settings key, no environment variable, no audit row per sweep and no data API (`apiEnabled: false`). `@objectstack/plugin-audit` leaves `sys_lifecycle_state` out of `sys_audit_log` and `sys_activity`, so a sweep writes no ledger row for its own clock. `OS_LIFECYCLE_DISABLED=1` still arms nothing. `os migrate organization-ownership` inventories the new table as deployment-level state with no organization column. diff --git a/packages/cli/src/utils/organization-ownership-inventory.ts b/packages/cli/src/utils/organization-ownership-inventory.ts index 3549c8493c1..ca211764af1 100644 --- a/packages/cli/src/utils/organization-ownership-inventory.ts +++ b/packages/cli/src/utils/organization-ownership-inventory.ts @@ -759,6 +759,14 @@ export const ORGANIZATION_OWNERSHIP_INVENTORY: readonly InventoryEntry[] = [ `${CENSUS_MAIN}: systemFields.tenant: false; ${D7}: the environment's trusted plugin-publisher keys, ` + 'read by key id under the system context, never by organization', }, + { + object: 'sys_lifecycle_state', + census: PMAIN, + fate: 'column-drop', + citation: + `${CENSUS_MAIN}: systemFields.tenant: false; ${D7}: the lifecycle sweeper's clock state, one row per ` + + 'datastore, written by the sweep under the system context, never by organization', + }, { object: 'sys_scim_connection_credential', census: P1, fate: 'attribution', citation: `${D3_ORG_ROWS}; ${D10_SINGLE}`, anchors: [] }, { object: 'sys_flow_credential', diff --git a/packages/objectql/src/lifecycle/lifecycle-service.test.ts b/packages/objectql/src/lifecycle/lifecycle-service.test.ts index f675f8cf68d..30b04587337 100644 --- a/packages/objectql/src/lifecycle/lifecycle-service.test.ts +++ b/packages/objectql/src/lifecycle/lifecycle-service.test.ts @@ -2638,6 +2638,277 @@ describe('LifecycleService timers', () => { }); }); +/** + * [#29918] The sweep's watermark: one engine-owned row (`sys_lifecycle_state`, + * id `lifecycle`) holding when the scheduled sweep last completed, so a kernel + * that reboots onto an existing store reads one row at the initial delay + * instead of re-sweeping. Every door the sweep can reach is counted, so "only + * the read" is a statement count, not an inference. + */ +describe('LifecycleService sweep watermark (#29918)', () => { + const INTERVAL = 5_000; + const DELAY = 1_000; + + function watermarkEngine(opts: { + row?: Record | null; + fresh?: boolean; + registered?: boolean; + findOneImpl?: () => Promise | null>; + deleteImpl?: (object: string, options: any) => any; + auditImpl?: () => Promise; + } = {}) { + const captured = captureEngine( + [{ name: 'sys_job_run', lifecycle: { class: 'telemetry', retention: { maxAge: '30d' } } }], + { deleteImpl: opts.deleteImpl }, + ); + const { engine } = captured; + const reads: Array<{ object: string; where: unknown; context: unknown }> = []; + const inserts: Array<{ object: string; data: Record; context: unknown }> = []; + const updates: Array<{ object: string; data: Record; context: unknown }> = []; + const audits: unknown[] = []; + const settingsReads: string[] = []; + let stored: Record | null = opts.row ?? null; + engine.registry.getObject = (name: string) => + name === 'sys_lifecycle_state' && opts.registered !== false ? { name } : undefined; + engine.wasDatastoreCreatedFromEmpty = () => opts.fresh === true; + engine.findOne = async (object: string, options: any) => { + reads.push({ object, where: options?.where, context: options?.context }); + return opts.findOneImpl ? opts.findOneImpl() : stored; + }; + engine.insert = async (object: string, data: any, options: any) => { + inserts.push({ object, data, context: options?.context }); + stored = { ...data }; + return data; + }; + engine.update = async (object: string, data: any, options: any) => { + updates.push({ object, data, context: options?.context }); + stored = { ...(stored ?? {}), ...data }; + return data; + }; + engine.inspectDanglingReferences = async (options: unknown) => { + audits.push(options); + if (opts.auditImpl) return opts.auditImpl(); + return { findings: [], scannedObjects: [], unreadableObjects: [] }; + }; + const getSettings = () => ({ + async get(namespace: string, key: string) { + settingsReads.push(`${namespace}.${key}`); + return { value: undefined }; + }, + }); + /** Every statement the sweep (and the watermark) could issue, by door. */ + const statements = () => + captured.deletes.length + reads.length + inserts.length + updates.length + audits.length + settingsReads.length; + return { ...captured, reads, inserts, updates, audits, settingsReads, getSettings, statements, row: () => stored }; + } + + const iso = (ms: number) => new Date(ms).toISOString(); + + async function bootAndReachInitialDelay( + harness: ReturnType, + extra: Partial[0]> = {}, + ) { + const svc = service(harness.engine, { + initialDelayMs: DELAY, + sweepIntervalMs: INTERVAL, + getSettings: harness.getSettings, + ...extra, + }); + svc.start(); + await vi.advanceTimersByTimeAsync(DELAY); + return svc; + } + + it('a reboot whose watermark is younger than the interval issues only the watermark read at the initial delay', async () => { + vi.useFakeTimers(); + try { + const harness = watermarkEngine({ row: { id: 'lifecycle', last_swept_at: iso(FIXED_NOW - (INTERVAL - 1)) } }); + const svc = await bootAndReachInitialDelay(harness); + + expect(harness.reads).toEqual([ + { object: 'sys_lifecycle_state', where: { id: 'lifecycle' }, context: expect.objectContaining({ isSystem: true }) }, + ]); + // Zero statements beyond the read: no reap, no settings, no audit, no write. + expect(harness.statements()).toBe(1); + expect(harness.deletes).toHaveLength(0); + + // The next sweep is due on the interval, and it advances the watermark. + await vi.advanceTimersByTimeAsync(INTERVAL); + expect(harness.deletes).toHaveLength(1); + expect(harness.audits).toHaveLength(1); + expect(harness.updates).toEqual([ + { + object: 'sys_lifecycle_state', + data: expect.objectContaining({ id: 'lifecycle', last_swept_at: iso(FIXED_NOW) }), + context: expect.objectContaining({ isSystem: true }), + }, + ]); + expect(harness.inserts).toHaveLength(0); + svc.stop(); + } finally { + vi.useRealTimers(); + } + }); + + it('a watermark one interval old or older sweeps at the initial delay and advances it', async () => { + vi.useFakeTimers(); + try { + for (const age of [INTERVAL, INTERVAL * 10]) { + const harness = watermarkEngine({ row: { id: 'lifecycle', last_swept_at: iso(FIXED_NOW - age) } }); + const svc = await bootAndReachInitialDelay(harness); + expect(harness.deletes).toHaveLength(1); + expect(harness.audits).toHaveLength(1); + // The boot read said the row exists, so the advance is one update, no second read. + expect(harness.reads).toHaveLength(1); + expect(harness.inserts).toHaveLength(0); + expect(harness.updates).toHaveLength(1); + expect(harness.row()).toMatchObject({ id: 'lifecycle', last_swept_at: iso(FIXED_NOW) }); + svc.stop(); + } + } finally { + vi.useRealTimers(); + } + }); + + it('no watermark row on an existing store sweeps at the initial delay and writes it', async () => { + vi.useFakeTimers(); + try { + const harness = watermarkEngine({ row: null }); + const svc = await bootAndReachInitialDelay(harness); + expect(harness.deletes).toHaveLength(1); + expect(harness.audits).toHaveLength(1); + expect(harness.reads).toHaveLength(1); + expect(harness.updates).toHaveLength(0); + expect(harness.inserts).toEqual([ + { + object: 'sys_lifecycle_state', + data: expect.objectContaining({ id: 'lifecycle', last_swept_at: iso(FIXED_NOW) }), + context: expect.objectContaining({ isSystem: true }), + }, + ]); + svc.stop(); + } finally { + vi.useRealTimers(); + } + }); + + it('a store this boot created still defers the first sweep, seeds the watermark without reading it, and sweeps one interval later', async () => { + vi.useFakeTimers(); + try { + const harness = watermarkEngine({ fresh: true }); + const svc = await bootAndReachInitialDelay(harness); + expect(harness.deletes).toHaveLength(0); + expect(harness.audits).toHaveLength(0); + expect(harness.settingsReads).toEqual([]); + expect(harness.reads).toHaveLength(0); + expect(harness.inserts).toEqual([ + { + object: 'sys_lifecycle_state', + data: expect.objectContaining({ id: 'lifecycle', last_swept_at: iso(FIXED_NOW) }), + context: expect.objectContaining({ isSystem: true }), + }, + ]); + + await vi.advanceTimersByTimeAsync(INTERVAL); + expect(harness.deletes).toHaveLength(1); + expect(harness.updates).toHaveLength(1); + svc.stop(); + } finally { + vi.useRealTimers(); + } + }); + + it('a sweep that fails a policy, or whose audit throws, leaves the watermark unchanged', async () => { + const before = { id: 'lifecycle', last_swept_at: iso(FIXED_NOW - INTERVAL * 10) }; + const reapFails = watermarkEngine({ + row: { ...before }, + deleteImpl: () => { + throw new Error('driver refused the delete'); + }, + }); + const failed = await service(reapFails.engine, { sweepIntervalMs: INTERVAL }).sweep(); + expect(failed.errors.length).toBeGreaterThan(0); + expect(reapFails.inserts).toHaveLength(0); + expect(reapFails.updates).toHaveLength(0); + expect(reapFails.row()).toEqual(before); + + const auditFails = watermarkEngine({ + row: { ...before }, + auditImpl: async () => { + throw new Error('audit could not read'); + }, + }); + const partial = await service(auditFails.engine, { sweepIntervalMs: INTERVAL }).sweep(); + expect(partial.errors).toEqual([]); + expect(auditFails.deletes).toHaveLength(1); + expect(auditFails.inserts).toHaveLength(0); + expect(auditFails.updates).toHaveLength(0); + expect(auditFails.row()).toEqual(before); + }); + + it('a watermark read that fails sweeps at the initial delay and says so at warn', async () => { + vi.useFakeTimers(); + try { + const warns: string[] = []; + const harness = watermarkEngine({ + findOneImpl: async () => { + throw new Error('connection refused'); + }, + }); + const svc = await bootAndReachInitialDelay(harness, { + logger: { info: () => {}, warn: (m: string) => warns.push(m), debug: () => {} }, + }); + expect(harness.deletes).toHaveLength(1); + expect(harness.audits).toHaveLength(1); + expect(warns.filter((m) => m.includes('watermark could not be read'))).toHaveLength(1); + svc.stop(); + } finally { + vi.useRealTimers(); + } + }); + + it('an engine that does not register sys_lifecycle_state is never asked for it, and sweeps at the initial delay', async () => { + vi.useFakeTimers(); + try { + const harness = watermarkEngine({ registered: false, row: { id: 'lifecycle', last_swept_at: iso(FIXED_NOW) } }); + const svc = await bootAndReachInitialDelay(harness); + expect(harness.deletes).toHaveLength(1); + expect(harness.reads).toHaveLength(0); + expect(harness.inserts).toHaveLength(0); + expect(harness.updates).toHaveLength(0); + svc.stop(); + } finally { + vi.useRealTimers(); + } + }); + + it('a direct sweep() sweeps whatever the watermark says, and advances it', async () => { + const harness = watermarkEngine({ row: { id: 'lifecycle', last_swept_at: iso(FIXED_NOW) } }); + await service(harness.engine, { sweepIntervalMs: INTERVAL }).sweep(); + expect(harness.deletes).toHaveLength(1); + expect(harness.audits).toHaveLength(1); + // Unknown presence on a direct call: one read to decide, then the update. + expect(harness.reads).toHaveLength(1); + expect(harness.updates).toHaveLength(1); + }); + + it('OS_LIFECYCLE_DISABLED still arms nothing: no watermark read, no write', async () => { + vi.useFakeTimers(); + process.env.OS_LIFECYCLE_DISABLED = '1'; + try { + const harness = watermarkEngine({ row: null }); + const svc = service(harness.engine, { initialDelayMs: DELAY, sweepIntervalMs: INTERVAL }); + svc.start(); + await vi.advanceTimersByTimeAsync(INTERVAL * 4); + expect(harness.statements()).toBe(0); + svc.stop(); + } finally { + delete process.env.OS_LIFECYCLE_DISABLED; + vi.useRealTimers(); + } + }); +}); + /** * [#4551] The read-only referential-integrity audit rides this sweep's clock. * diff --git a/packages/objectql/src/lifecycle/lifecycle-service.ts b/packages/objectql/src/lifecycle/lifecycle-service.ts index 4140b04c1aa..ba3f2b02a37 100644 --- a/packages/objectql/src/lifecycle/lifecycle-service.ts +++ b/packages/objectql/src/lifecycle/lifecycle-service.ts @@ -84,6 +84,22 @@ export const DEFAULT_LIFECYCLE_SWEEP_MS = 3_600_000; * and keeps short-lived test kernels from ever sweeping. */ export const DEFAULT_LIFECYCLE_INITIAL_DELAY_MS = 60_000; +/** + * [#29918] The engine-owned object that holds the sweep's clock state: + * one row per datastore, carrying when the scheduled sweep last completed + * (`last_swept_at`). Declared by `@objectstack/platform-objects` + * (`sys-lifecycle-state.object.ts`); a kernel that does not register it is + * one that cannot say, and sweeps exactly as before. + * + * Internal state, not a setting: no settings key, no env override, no audit + * row per sweep, no data API. See {@link LifecycleService.start} for the read + * and {@link LifecycleService.sweep} for the write. + */ +export const LIFECYCLE_STATE_OBJECT = 'sys_lifecycle_state'; + +/** [#29918] The row id of this service's clock in {@link LIFECYCLE_STATE_OBJECT}. */ +export const LIFECYCLE_STATE_ROW_ID = 'lifecycle'; + /** Minimal engine surface the service needs — duck-typed for tests. */ export interface LifecycleEngineLike { registry: { @@ -135,8 +151,25 @@ export interface LifecycleEngineLike { * first sweep runs at the initial delay exactly as before. */ wasDatastoreCreatedFromEmpty?(): boolean; + /** + * [#29918] Single-row read of the sweep's clock state + * ({@link LIFECYCLE_STATE_OBJECT}), by primary key. `findOne`, not `find` + * with `limit: 1`: it is the single-row route, and says what the read is. + * Optional with {@link insert} and {@link update}: an engine missing any of + * the three keeps no watermark, and its first sweep runs at the initial + * delay exactly as before. + */ + findOne?(object: string, options: Record): Promise | null>; + /** [#29918] Writes the clock-state row the first time. See {@link findOne}. */ + insert?(object: string, data: Record, options?: Record): Promise; + /** [#29918] Advances the clock-state row. See {@link findOne}. */ + update?(object: string, data: Record, options?: Record): Promise; } +/** [#29918] The engine surface the watermark needs, once it is known present. */ +type WatermarkEngine = LifecycleEngineLike & + Required>; + export interface LifecycleObjectLike { name: string; lifecycle?: Lifecycle; @@ -501,6 +534,16 @@ export class LifecycleService { */ private abort: { aborted: boolean } = { aborted: false }; + /** + * [#29918] Whether the clock-state row exists in the store: `undefined` + * until a read or a write has said, so an advance knows whether to insert + * or update without a second read. + */ + private watermarkRowExists: boolean | undefined; + + /** [#29918] A watermark write that fails is logged at `warn` once, then at `debug`. */ + private watermarkWriteWarned = false; + constructor(private readonly opts: LifecycleServiceOptions) { this.now = opts.now ?? (() => Date.now()); } @@ -524,7 +567,9 @@ export class LifecycleService { * * [#29918] On a datastore this process created from empty, the first sweep * is due one interval later instead of at the initial delay: see - * {@link firstSweepDueAtInitialDelay}. The interval is armed either way. */ + * {@link firstSweepDueAtInitialDelay}. On a store that was there before + * this boot, the sweep's watermark decides: see {@link runFirstScheduledSweep}. + * The interval is armed either way. */ start(): void { if (!this.enabled || this.timer || this.initialTimer) return; // A fresh bit per armed run — the one a previous sweep captured stays @@ -534,13 +579,147 @@ export class LifecycleService { const initial = this.opts.initialDelayMs ?? DEFAULT_LIFECYCLE_INITIAL_DELAY_MS; this.initialTimer = setTimeout(() => { this.initialTimer = undefined; - if (this.firstSweepDueAtInitialDelay()) void this.sweep(); + void this.runFirstScheduledSweep(interval); this.timer = setInterval(() => void this.sweep(), interval); this.timer.unref?.(); }, initial); this.initialTimer.unref?.(); } + /** + * [#29918] The scheduler's first tick, combining the two answers to "is a + * sweep due yet?". + * + * 1. A store this boot created from empty ({@link firstSweepDueAtInitialDelay}) + * defers one interval, as before, and seeds the watermark at now: no row + * there can be older than this boot, so a sweep now would find nothing, + * and a kernel that reboots onto the store inside the interval then + * skips too. + * 2. Otherwise the watermark row is read, once ({@link watermarkIsYoung}). + * Younger than the interval ⇒ that sweep is already done and the next is + * due on the interval; the read is the only statement issued. + * 3. Otherwise (an old or absent watermark, an engine that cannot say, a + * read that failed) the first sweep runs now, as before #29918. + * + * Only the scheduler asks: a direct {@link sweep} call always sweeps. Never + * throws, because the timer enters it as `void`. + */ + private async runFirstScheduledSweep(intervalMs: number): Promise { + if (this.stopped) return; + if (!this.firstSweepDueAtInitialDelay()) { + const engine = this.watermarkEngine(this.opts.getEngine()); + await this.writeWatermark(engine, new Date(this.now()).toISOString(), false); + return; + } + if (await this.watermarkIsYoung(intervalMs)) return; + await this.sweep(); + } + + /** + * [#29918] The engine, when it can keep the watermark: it reads, inserts + * and updates, and its registry has {@link LIFECYCLE_STATE_OBJECT}. Asked + * through `registry.getObject` so an unregistered object is never read: + * the engine would refuse it with `OBJECT_NOT_FOUND` before any driver. + */ + private watermarkEngine(engine: LifecycleEngineLike | undefined): WatermarkEngine | undefined { + if (!engine) return undefined; + if (typeof engine.findOne !== 'function' || typeof engine.insert !== 'function' || typeof engine.update !== 'function') { + return undefined; + } + let registered = false; + try { + registered = Boolean(engine.registry?.getObject?.(LIFECYCLE_STATE_OBJECT)); + } catch { + registered = false; + } + return registered ? (engine as WatermarkEngine) : undefined; + } + + /** + * [#29918] Whether the last completed sweep is younger than one interval. + * + * Every answer but a readable, past, younger-than-the-interval timestamp is + * `false`, which means "sweep": no engine that can keep a watermark, no row, + * an unparsable value, a value in the future (the clock moved back), and a + * read that failed. A failed read is logged at `warn`: it happens once per + * armed run, so that is once. + */ + private async watermarkIsYoung(intervalMs: number): Promise { + const engine = this.watermarkEngine(this.opts.getEngine()); + if (!engine) return false; + let row: Record | null; + try { + row = await engine.findOne(LIFECYCLE_STATE_OBJECT, { + where: { id: LIFECYCLE_STATE_ROW_ID }, + context: { ...SYSTEM_CTX }, + }); + } catch (err) { + this.opts.logger.warn( + `[lifecycle] the sweep watermark could not be read (${(err as Error)?.message ?? err}); ` + + 'sweeping at the initial delay as if none were recorded', + ); + return false; + } + this.watermarkRowExists = Boolean(row); + if (!row) return false; + const at = Date.parse(String(row.last_swept_at ?? '')); + const age = this.now() - at; + if (!Number.isFinite(at) || age < 0 || age >= intervalMs) return false; + this.opts.logger.debug?.( + `[lifecycle] the last sweep completed ${Math.round(age / 1000)}s ago, inside the interval; ` + + 'the first sweep runs on the interval', + ); + return true; + } + + /** + * [#29918] Record `at` as the last completed sweep. `exists` says whether + * the row is there; `undefined` asks the store once, then remembers. + * + * A write that fails is a functional degradation, not a durability one: + * the rows the sweep reaped are reaped, and the only cost is that the next + * boot sweeps again because the watermark is older than it should be. So + * it is logged at `warn` the first time and at `debug` after, never thrown. + */ + private async writeWatermark( + engine: WatermarkEngine | undefined, + at: string, + exists: boolean | undefined = this.watermarkRowExists, + ): Promise { + if (!engine) return; + try { + let present = exists; + if (present === undefined) { + const row = await engine.findOne(LIFECYCLE_STATE_OBJECT, { + where: { id: LIFECYCLE_STATE_ROW_ID }, + context: { ...SYSTEM_CTX }, + }); + present = Boolean(row); + } + // Only the clock's own columns: the audit family (`created_at`, + // `updated_at`) is the engine's to stamp on its write path. + const row = { id: LIFECYCLE_STATE_ROW_ID, last_swept_at: at }; + if (present) { + await engine.update(LIFECYCLE_STATE_OBJECT, row, { context: { ...SYSTEM_CTX } }); + } else { + await engine.insert(LIFECYCLE_STATE_OBJECT, row, { context: { ...SYSTEM_CTX } }); + } + this.watermarkRowExists = true; + } catch (err) { + // The row's presence is no longer known: the next write asks again. + this.watermarkRowExists = undefined; + const msg = + `[lifecycle] the sweep watermark could not be written (${(err as Error)?.message ?? err}); ` + + 'nothing was lost, but the next boot sweeps again at the initial delay'; + if (this.watermarkWriteWarned) { + this.opts.logger.debug?.(msg); + } else { + this.watermarkWriteWarned = true; + this.opts.logger.warn(msg); + } + } + } + /** * [#29918] Whether the scheduled first sweep runs at the initial delay. * @@ -865,7 +1044,16 @@ export class LifecycleService { // // Failure is isolated like every other leg: an audit that cannot run must // never cost a sweep its reaping. - await this.auditReferences(engine, report); + const audited = await this.auditReferences(engine, report); + + // [#29918] Advance the watermark only for a sweep that COMPLETED: no + // policy error, the audit leg ran, and no teardown cut it short. A + // failed or partial sweep leaves the last good time in place, so the + // next boot sweeps. Stamped with the sweep's start: the cutoffs were + // taken from it. + if (audited && report.errors.length === 0 && !this.stopped) { + await this.writeWatermark(this.watermarkEngine(engine), report.at); + } if (report.swept.length > 0 || report.errors.length > 0 || report.alerts.length > 0) { // ADR-0057 §3.3: cleanup must not re-feed the tables it drains — one @@ -894,30 +1082,36 @@ export class LifecycleService { * could not run has applied no policy either way. It logs instead, and its * own report already carries `unreadableObjects` / `undetermined` so a * partial run is never mistaken for a clean one. + * + * [#29918] Answers whether the leg is complete — `true` when it ran, or was + * never meant to (switched off, or an engine without the audit) — so a sweep + * whose audit threw or was cut short does not advance the watermark. */ private async auditReferences( engine: LifecycleEngineLike, report: LifecycleSweepReport, - ): Promise { + ): Promise { const cfg = this.opts.referenceAudit; - if (cfg?.enabled === false) return; - if (typeof engine.inspectDanglingReferences !== 'function') return; + if (cfg?.enabled === false) return true; + if (typeof engine.inspectDanglingReferences !== 'function') return true; // [#4747] Not a config switch keyed on "is this a one-shot process" — the // audit stays wired on every host, exactly as #4551 intends. What it gets // is the teardown bit: it reads while the engine is live and stops when the // engine is going away, so `unreadableObjects` keeps meaning "the // datasource refused" and nothing else. - if (this.stopped) return; + if (this.stopped) return false; try { const { enabled: _enabled, ...auditOptions } = cfg ?? {}; report.danglingReferences = await engine.inspectDanglingReferences({ ...auditOptions, signal: this.abort, }); + return true; } catch (err) { this.opts.logger.warn( `[lifecycle] reference audit failed (${(err as Error)?.message ?? err})`, ); + return false; } } diff --git a/packages/platform-objects/src/deployment-level-no-tenant-column.test.ts b/packages/platform-objects/src/deployment-level-no-tenant-column.test.ts index 3469020c1fd..a7233f9a531 100644 --- a/packages/platform-objects/src/deployment-level-no-tenant-column.test.ts +++ b/packages/platform-objects/src/deployment-level-no-tenant-column.test.ts @@ -5,6 +5,7 @@ import { resolveInjectedSystemColumns } from '@objectstack/spec/data'; import { PLATFORM_CAPABILITIES } from '@objectstack/spec/security'; import { SysJob, SysJobRun, SysJobQueue } from './audit/index.js'; import { + SysLifecycleState, SysMigration, SysMigrationJournal, SysPlatformSettingAudit, @@ -48,6 +49,8 @@ const DEPLOYMENT_LEVEL = [ ['sys_platform_setting_audit', SysPlatformSettingAudit], // [ADR-0025 §3.5 M1] The environment's trusted plugin-publisher keys. ['sys_trusted_key', SysTrustedKey], + // [ADR-0057] The lifecycle sweeper's clock state: one row per datastore. + ['sys_lifecycle_state', SysLifecycleState], ] as const; describe('[ADR-0131 D7] deployment-level plumbing: no tenant column, platform-only reads', () => { diff --git a/packages/platform-objects/src/plugin.test.ts b/packages/platform-objects/src/plugin.test.ts index efb84ef80b0..5385eea2abf 100644 --- a/packages/platform-objects/src/plugin.test.ts +++ b/packages/platform-objects/src/plugin.test.ts @@ -10,7 +10,7 @@ import { describe, it, expect } from 'vitest'; // the predicate into a package that depends on neither side. import { assertEngineFindOnePredicate, assertEngineUpdateDispatch } from '@objectstack/metadata-core'; import { PlatformObjectsPlugin } from './plugin.js'; -import { SysExchangeRate, SysMetadataActivation, SysMigration, SysMigrationJournal, SysSecret, SysTrustedKey } from './system/index.js'; +import { SysExchangeRate, SysLifecycleState, SysMetadataActivation, SysMigration, SysMigrationJournal, SysSecret, SysTrustedKey } from './system/index.js'; /** * Hand-rolled fake PluginContext (mirrors the service-plugin tests) — this @@ -46,7 +46,7 @@ function makeCtx() { } describe('PlatformObjectsPlugin: platform-infrastructure object registration (#4243, #4270)', () => { - it('registers SysMigration, SysMigrationJournal, SysSecret, SysExchangeRate and SysTrustedKey through the manifest service', async () => { + it('registers SysMigration, SysMigrationJournal, SysSecret, SysExchangeRate, SysTrustedKey and SysLifecycleState through the manifest service', async () => { const ctx = makeCtx(); const manifests: any[] = []; ctx.registerService('manifest', { register: (m: any) => manifests.push(m) }); @@ -56,7 +56,7 @@ describe('PlatformObjectsPlugin: platform-infrastructure object registration (#4 const infra = manifests.find((m) => m.id === 'com.objectstack.platform-objects'); expect(infra).toBeDefined(); expect(infra.scope).toBe('system'); - expect(infra.objects).toEqual([SysMigration, SysMigrationJournal, SysSecret, SysExchangeRate, SysTrustedKey]); + expect(infra.objects).toEqual([SysMigration, SysMigrationJournal, SysSecret, SysExchangeRate, SysTrustedKey, SysLifecycleState]); expect(infra.objects.map((o: any) => o.name)).toEqual([ 'sys_migration', // ADR-0119 D2 (#4617). Registered UNCONDITIONALLY, alongside the flag @@ -73,8 +73,11 @@ describe('PlatformObjectsPlugin: platform-infrastructure object registration (#4 // keys, read by `os plugin install` from a one-shot boot that composes // this plugin and nothing that would register it otherwise. 'sys_trusted_key', + // [ADR-0057] The lifecycle sweeper's clock state, read and written only + // by the engine's `LifecycleService`, which reads the store it sweeps. + 'sys_lifecycle_state', ]); - // ⛔ The five above must NOT acquire the ledger's routing triple. They + // ⛔ The six above must NOT acquire the ledger's routing triple. They // ride the project database today; moving them is a different question. expect(infra.defaultDatasource).toBeUndefined(); }); diff --git a/packages/platform-objects/src/plugin.ts b/packages/platform-objects/src/plugin.ts index 83152351592..3656cdb7f01 100644 --- a/packages/platform-objects/src/plugin.ts +++ b/packages/platform-objects/src/plugin.ts @@ -8,6 +8,7 @@ import { SysMigrationJournal } from './system/sys-migration-journal.object.js'; import { SysSecret } from './system/sys-secret.object.js'; import { SysExchangeRate } from './system/sys-exchange-rate.object.js'; import { SysTrustedKey } from './system/sys-trusted-key.object.js'; +import { SysLifecycleState } from './system/sys-lifecycle-state.object.js'; import { attestFreshDatastore } from './system/migration-flag.js'; import type { II18nService, @@ -56,6 +57,12 @@ import { SEED_SETTLEMENT_SERVICE } from '@objectstack/spec/contracts'; * and ADR-0025 M1's boot-time plugin loader in the runtime is the second * reader (not built yet). No capability plugin owns plugin distribution, * so it rides with the other platform infrastructure (ADR-0029 D1/D2). + * - **`sys_lifecycle_state`** — the ObjectQL lifecycle sweeper's clock + * state (ADR-0057): when the scheduled sweep last completed on this + * datastore, so a reboot onto an existing store reads one row instead of + * re-sweeping within a minute. Its only reader and writer is the engine's + * own `LifecycleService`, which treats a kernel without this object as + * one that cannot say and sweeps as before. * - **`sys_metadata_activation`** — the packaged-metadata activation * ledger (ADR-0126 §4). Same story a third time, and the ruling that * moved it here is recorded verbatim on #12359 (maintainer, 2026-08-26, @@ -202,7 +209,7 @@ export class PlatformObjectsPlugin { version: this.version, type: 'plugin', scope: 'system', - objects: [SysMigration, SysMigrationJournal, SysSecret, SysExchangeRate, SysTrustedKey], + objects: [SysMigration, SysMigrationJournal, SysSecret, SysExchangeRate, SysTrustedKey, SysLifecycleState], }); ctx?.getService?.('manifest')?.register?.({ ...ACTIVATION_LEDGER_MANIFEST, diff --git a/packages/platform-objects/src/system/index.ts b/packages/platform-objects/src/system/index.ts index 9849501f4ca..8e91f32b711 100644 --- a/packages/platform-objects/src/system/index.ts +++ b/packages/platform-objects/src/system/index.ts @@ -23,6 +23,8 @@ export { SysSettingAudit } from './sys-setting-audit.object.js'; export { SysPlatformSettingAudit } from './sys-platform-setting-audit.object.js'; export { SysMigration } from './sys-migration.object.js'; export { SysMigrationJournal } from './sys-migration-journal.object.js'; +// [ADR-0057] The lifecycle sweeper's clock state, tenant-less. +export { SysLifecycleState } from './sys-lifecycle-state.object.js'; // Multi-currency M1-1 (epic #8345): the dated rate table the write-time // home-currency projection reads. export { diff --git a/packages/platform-objects/src/system/sys-lifecycle-state.object.ts b/packages/platform-objects/src/system/sys-lifecycle-state.object.ts new file mode 100644 index 00000000000..d28da14a1b4 --- /dev/null +++ b/packages/platform-objects/src/system/sys-lifecycle-state.object.ts @@ -0,0 +1,94 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { ObjectSchema, Field } from '@objectstack/spec/data'; + +/** + * sys_lifecycle_state — the lifecycle sweeper's own clock state (ADR-0057). + * + * One row per CLOCK, keyed by a well-known id. The only clock today is the + * ObjectQL `LifecycleService` sweep (row id `lifecycle`), which the + * dangling-reference audit rides. The row holds when that sweep last + * COMPLETED on this datastore, so a kernel that reboots onto an existing store + * reads one row at the initial delay and skips a sweep the interval has not + * yet come due for, instead of re-reading every declared object, the + * governance settings and every audited reference within a minute of each + * boot. + * + * Internal state, not a setting: ⛔ no settings key, no env override, no + * audit row per sweep, no data API (`apiEnabled: false` below). The sole + * writer is the sweeper itself, under a system context, after a sweep that + * finished with no policy error; a failed or partial sweep leaves the row as + * it was. The sole reader is the same sweeper, once per boot. A missing row, + * a missing object (a kernel that does not compose this package) or a read + * that fails all mean "sweep": the row can only ever save work, never + * withhold it. + * + * Row identity: one row per datastore. The sweep is per store, not per + * organization — it reaps every tenant's rows in one pass — so the row + * describes the store it lives in, and carries no tenant column (see + * `systemFields`). + * + * Lifecycle: **no `lifecycle` block on purpose.** The absent block is the + * back-compat `record` class — never swept. Reaping the row that records the + * sweep would only make the next boot sweep again. + * + * Registered by `PlatformObjectsPlugin` (`../plugin.ts`) beside the other + * deployment-level plumbing it resembles (`sys_migration`). + * + * @namespace sys + */ +export const SysLifecycleState = ObjectSchema.create({ + name: 'sys_lifecycle_state', + label: 'Lifecycle State', + pluralLabel: 'Lifecycle States', + icon: 'timer', + isSystem: true, + managedBy: 'engine-owned', + // [ADR-0131 D7] Deployment-level state: NO tenant column. The sole writer is + // the lifecycle sweeper, under a system context, and one sweep covers every + // organization's rows at once, so there is no organization to stamp. Who may + // read is object permission (D7): the platform-only capability below. + systemFields: { tenant: false }, + requiredPermissions: ['manage_platform_settings'], + description: + 'Lifecycle sweeper clock state: when the scheduled retention sweep last completed on this datastore. One row per clock.', + nameField: 'id', // [ADR-0079] canonical primary-title pointer + highlightFields: ['id', 'last_swept_at'], + + fields: { + id: Field.text({ + label: 'Clock ID', + required: true, + readonly: true, + description: 'Well-known clock id. `lifecycle` is the ObjectQL lifecycle sweep. One row per clock.', + }), + + last_swept_at: Field.datetime({ + label: 'Last Swept At', + required: true, + readonly: true, + description: + 'When the sweep last completed with no policy error on this datastore. On a datastore the ' + + 'process created from empty it is the time the store was found empty, since no row there ' + + 'can be older.', + }), + + created_at: Field.datetime({ + label: 'Created At', + readonly: true, + }), + + updated_at: Field.datetime({ + label: 'Updated At', + readonly: true, + }), + }, + + enable: { + trackHistory: false, + searchable: false, + // Internal clock state — no data API at all. The sweeper reads and writes + // it in process, under a system context. + apiEnabled: false, + }, +}); diff --git a/packages/plugins/plugin-audit/src/audit-writers.test.ts b/packages/plugins/plugin-audit/src/audit-writers.test.ts index d9a462fbd00..6d5a52c47ca 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.test.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.test.ts @@ -323,6 +323,29 @@ describe('audit writers — operational plumbing is excluded (#5193, ADR-0057 D5 } }); + it('writes NO audit/activity row for the lifecycle sweeper\'s watermark, first insert or hourly update (ADR-0057 §3.3)', async () => { + // `LifecycleService` inserts the `sys_lifecycle_state` row after its first + // completed sweep and updates it after every one since. Audited, each + // sweep fed the ledger a row the next sweep archives: cleanup re-feeding + // the table it drains. + const { engine, fire, created } = makeEngine({ ...SCHEMA, sys_lifecycle_state: ['id', 'last_swept_at'] }); + installAuditWriters(engine as any, 'test.audit'); + await fire('afterInsert', { + object: 'sys_lifecycle_state', + input: { id: 'lifecycle', last_swept_at: '2026-11-06T00:00:00.000Z' }, + result: { id: 'lifecycle', last_swept_at: '2026-11-06T00:00:00.000Z' }, + session: { isSystem: true }, + }); + await fire('afterUpdate', { + object: 'sys_lifecycle_state', + input: { id: 'lifecycle', last_swept_at: '2026-11-06T01:00:00.000Z' }, + previous: { id: 'lifecycle', last_swept_at: '2026-11-06T00:00:00.000Z' }, + result: { id: 'lifecycle', last_swept_at: '2026-11-06T01:00:00.000Z' }, + session: { isSystem: true }, + }); + expect(created).toEqual([]); + }); + it('pays no before-phase snapshot read — for the skipped object OR the business one (#6656)', async () => { // [#6656] This case used to assert the saving for `sys_job_queue` only, // with `crm_lead` as the CONTROL that still paid — because #5860 could diff --git a/packages/plugins/plugin-audit/src/audit-writers.ts b/packages/plugins/plugin-audit/src/audit-writers.ts index 1ac08c8b78d..91547729802 100644 --- a/packages/plugins/plugin-audit/src/audit-writers.ts +++ b/packages/plugins/plugin-audit/src/audit-writers.ts @@ -298,6 +298,14 @@ const SKIP_OBJECTS = new Set([ // and keep their compliance value, so they stay audited (see #5202). 'sys_upload_session', // chunked-upload progress (1 write per chunk) 'ai_traces', // LLM trace telemetry + // [ADR-0057 §3.3] The lifecycle sweeper's own clock state (one row per + // datastore, `managedBy: 'engine-owned'`), written ONLY by objectql's + // `LifecycleService` under SYSTEM_CTX after every completed sweep. Audited, + // each sweep would feed `sys_audit_log` and `sys_activity` the very rows the + // next sweep archives or reaps: cleanup re-feeding the tables it drains. + // Not an act anyone performed; the ruling that created the object forbids + // an audit row per sweep. + 'sys_lifecycle_state', // sweep watermark (1 write per sweep) ]); /** diff --git a/packages/spec/src/system/constants/platform-object-names.ts b/packages/spec/src/system/constants/platform-object-names.ts index f6a95cac013..4ca2fff47ed 100644 --- a/packages/spec/src/system/constants/platform-object-names.ts +++ b/packages/spec/src/system/constants/platform-object-names.ts @@ -60,6 +60,7 @@ export const PLATFORM_OBJECTS_BY_PACKAGE: Readonly